Duplicati: flag warnings run after run, and forecast how much room backups take
A job whose last three runs in a row all finished with warnings is flagged and raises an alert (on by default, switchable off per job). The size of the destination, and its quota where the storage says, are read from the reports and kept with each run; the Stored column shows the growth and, where the room is known, when it will be full, with an alert inside the disk-forecast window. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
333146e8e3
commit
4f077d4f4a
15 files changed
+10115
-14
No files matched your search
@@ -0,0 +1,263 @@
|
||||
/**
|
||||
* Duplicati, the slower things: a job whose last runs in a row all had warnings, and how much room the backups take where they are kept (read from what Duplicati reports, worked into a growth and a
|
||||
* forecast, and an alert when a destination with a known size will be full within the forecast window).
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-dupfollow-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, duplicatiSources, duplicatiBackups, duplicatiRuns } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const d = await import("../src/services/duplicati.js");
|
||||
const { duplicatiRouter } = await import("../src/routes/duplicati.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const DAY = 86_400_000;
|
||||
const NOW = new Date("2026-11-10T12:00:00.000Z");
|
||||
const daysAgo = (n: number) => new Date(NOW.getTime() - n * DAY).toISOString();
|
||||
|
||||
const report = (over: Record<string, unknown> = {}, backend: Record<string, unknown> | null = { KnownFileSize: 5_000_000_000, TotalQuotaSpace: 0 }, name = "Documents") =>
|
||||
JSON.stringify({
|
||||
Data: { MainOperation: "Backup", ParsedResult: "Success", EndTime: "2026-10-23T03:35:44.1Z", BeginTime: "2026-10-23T03:30:31.4Z", Duration: "00:05:12.7", Warnings: [], Errors: [], ...(backend ? { BackendStatistics: backend } : {}), ...over },
|
||||
Extra: { OperationName: "Backup", "backup-name": name },
|
||||
});
|
||||
|
||||
describe("what the report says about the destination", () => {
|
||||
test("JSON: the size where the backups are kept, and the room there is when its storage says", () => {
|
||||
const r = d.parseReport(report({}, { KnownFileSize: 12_345_678, TotalQuotaSpace: 500_000_000_000, FreeQuotaSpace: 1 }))!;
|
||||
assert.deepEqual([r.destinationBytes, r.quotaBytes], [12_345_678, 500_000_000_000]);
|
||||
});
|
||||
|
||||
test("a quota of zero or less is not said, a size below zero is not said, and a report without the section says neither", () => {
|
||||
assert.deepEqual(d.destinationOf(100, 0), { destinationBytes: 100, quotaBytes: null });
|
||||
assert.deepEqual(d.destinationOf(100, -1), { destinationBytes: 100, quotaBytes: null });
|
||||
assert.deepEqual(d.destinationOf(-5, 10), { destinationBytes: null, quotaBytes: 10 });
|
||||
assert.deepEqual(d.destinationOf(null, null), { destinationBytes: null, quotaBytes: null });
|
||||
const none = d.parseReport(report({}, null))!;
|
||||
assert.deepEqual([none.destinationBytes, none.quotaBytes], [null, null]);
|
||||
const zero = d.parseReport(report({}, { KnownFileSize: 0, TotalQuotaSpace: 0 }))!;
|
||||
assert.deepEqual([zero.destinationBytes, zero.quotaBytes], [0, null], "an empty destination is a size, not a missing one");
|
||||
});
|
||||
|
||||
test("the plain text form: the bytes come first, the readable size after", () => {
|
||||
const text = ["MainOperation: Backup", "ParsedResult: Success", "Duration: 00:01:00", "BackendStatistics:", " KnownFileSize: 5811709 (5.54 MB)", " TotalQuotaSpace: 107374182400 (100.00 GB)", " FreeQuotaSpace: 0", "backup-name: Docs"].join("\n");
|
||||
const r = d.parseReport(text)!;
|
||||
assert.deepEqual([r.destinationBytes, r.quotaBytes], [5_811_709, 107_374_182_400]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("warnings in a row", () => {
|
||||
test("counted from the newest run; a success, a failure or an unknown word ends it", () => {
|
||||
assert.equal(d.warningStreak([]), 0);
|
||||
assert.equal(d.warningStreak(["Warning", "Warning", "Warning", "Success", "Warning"]), 3);
|
||||
assert.equal(d.warningStreak(["warning", "WARNING"]), 2);
|
||||
assert.equal(d.warningStreak(["Success", "Warning", "Warning"]), 0);
|
||||
assert.equal(d.warningStreak(["Warning", "Error", "Warning"]), 1);
|
||||
assert.equal(d.STREAK_RUNS, 3);
|
||||
});
|
||||
});
|
||||
|
||||
describe("growth", () => {
|
||||
const steady = (days: number, perDay: number, start = 1e9, every = 1) => Array.from({ length: Math.floor(days / every) + 1 }, (_, i) => ({ t: NOW.getTime() - (days - i * every) * DAY, v: start + i * every * perDay }));
|
||||
|
||||
test("a steady climb: bytes a day and a month, how well a line fits, and when a known quota is full", () => {
|
||||
const g = d.growthOf(steady(20, 50_000_000), 5e9)!;
|
||||
assert.equal(g.perDay, 50_000_000);
|
||||
assert.equal(g.perMonth, 1_500_000_000);
|
||||
assert.equal(g.fit, 1);
|
||||
assert.equal(g.spanDays, 20);
|
||||
assert.equal(g.latest, 2e9);
|
||||
assert.equal(g.daysToFull, 60, "(5 GB − 2 GB) at 50 MB a day");
|
||||
});
|
||||
|
||||
test("no quota, nothing to fill; shrinking, nothing to fill; already over, nothing to fill", () => {
|
||||
assert.equal(d.growthOf(steady(20, 50_000_000), null)!.daysToFull, null);
|
||||
const shrinking = d.growthOf(steady(20, -20_000_000, 5e9), 9e9)!;
|
||||
assert.ok(shrinking.perDay < 0);
|
||||
assert.equal(shrinking.daysToFull, null);
|
||||
assert.equal(d.growthOf(steady(20, 50_000_000), 1e9)!.daysToFull, null);
|
||||
});
|
||||
|
||||
test("not enough to go on: too few readings, or too short a time", () => {
|
||||
assert.equal(d.growthOf(steady(20, 1, 1e9, 10), null), null, "three readings");
|
||||
assert.equal(d.growthOf(steady(5, 1_000, 1e9, 1), null), null, "five days");
|
||||
assert.equal(d.growthOf([], null), null);
|
||||
const flat = d.growthOf(steady(20, 0), 5e9)!;
|
||||
assert.deepEqual([flat.perDay, flat.fit, flat.daysToFull], [0, 0, null], "a size that does not move has no fit and nothing to fill");
|
||||
});
|
||||
|
||||
test("a jagged size (a compaction now and then) fits badly", () => {
|
||||
const jagged = steady(30, 0).map((p, i) => ({ ...p, v: 1e9 + (i % 5 === 0 ? 0 : 3e8) * (i % 3) }));
|
||||
assert.ok(d.growthOf(jagged, 5e9)!.fit < d.GROWTH_MIN_FIT);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what a report does to the destination, and the alerts from it", () => {
|
||||
let sourceId = 0;
|
||||
let token = "";
|
||||
before(async () => {
|
||||
const made = await d.createSource({ name: "Jackie", serverId: null, createdBy: "Ada" });
|
||||
sourceId = made.row.id;
|
||||
token = made.token;
|
||||
});
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const backupId = async (name: string) => (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, name)))[0].id;
|
||||
|
||||
test("it is written on the job and on the run; a report that does not say leaves what was known", async () => {
|
||||
await d.recordReport(token, report({}, { KnownFileSize: 7_000_000_000, TotalQuotaSpace: 9_000_000_000 }, "Docs"), new Date("2026-11-01T04:00:00Z"));
|
||||
const id = await backupId("Docs");
|
||||
let [b] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id));
|
||||
assert.deepEqual([b.destinationBytes, b.quotaBytes], [7_000_000_000, 9_000_000_000]);
|
||||
assert.equal((await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, id)))[0].destinationBytes, 7_000_000_000);
|
||||
|
||||
await d.recordReport(token, report({}, null, "Docs"), new Date("2026-11-02T04:00:00Z"));
|
||||
[b] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id));
|
||||
assert.deepEqual([b.destinationBytes, b.quotaBytes], [7_000_000_000, 9_000_000_000], "kept");
|
||||
const runs = await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, id));
|
||||
assert.deepEqual(runs.map((r) => r.destinationBytes).sort(), [7_000_000_000, null], "the run itself says nothing");
|
||||
});
|
||||
|
||||
/** A job with a history made directly: the results newest last, one a day, and (optionally) a growing size. */
|
||||
async function jobWith(name: string, results: string[], size?: { start: number; perDay: number; quota: number | null }, flags: Partial<typeof duplicatiBackups.$inferInsert> = {}) {
|
||||
const [b] = await db
|
||||
.insert(duplicatiBackups)
|
||||
.values({ sourceId, name, lastResult: results[results.length - 1], lastSuccessAt: daysAgo(0.5), lastReportAt: daysAgo(0.5), warnings: results[results.length - 1] === "Warning" ? 2 : 0, warningMessages: JSON.stringify(["locked file", "another"]), quotaBytes: size?.quota ?? null, ...flags })
|
||||
.returning();
|
||||
await db.insert(duplicatiRuns).values(results.map((result, i) => ({ backupId: b.id, at: daysAgo(results.length - 1 - i + 0.5), result, warnings: result === "Warning" ? 1 : 0, errors: 0, destinationBytes: size ? size.start + i * size.perDay : null })));
|
||||
return b;
|
||||
}
|
||||
const problemsOf = async (b: typeof duplicatiBackups.$inferSelect, forecastDays = 14) => {
|
||||
const [fresh] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, b.id));
|
||||
const [source] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, sourceId));
|
||||
const byBackup = await d.analyseBackups([fresh], NOW);
|
||||
return d.duplicatiProblems([source], [fresh], NOW.getTime(), { byBackup, forecastDays }).filter((p) => p.key.endsWith(`:${b.id}`) || p.key.includes(`:${b.id}:`));
|
||||
};
|
||||
|
||||
test("three warnings in a row is said once, with what the last said; two is not, and a success after them ends it", async () => {
|
||||
const three = await jobWith("Three", ["Success", "Warning", "Warning", "Warning"]);
|
||||
const p = await problemsOf(three);
|
||||
assert.deepEqual(p.map((x) => [x.key, x.kind, x.severity]), [[`duplicati-streak:${three.id}`, "streak", "warning"]]);
|
||||
assert.match(p[0].message, /“Three” on Jackie has finished with warnings 3 runs in a row — the last said: locked file/);
|
||||
|
||||
const two = await jobWith("Two", ["Success", "Warning", "Warning"]);
|
||||
assert.deepEqual(await problemsOf(two), []);
|
||||
const healed = await jobWith("Healed", ["Warning", "Warning", "Warning", "Success"]);
|
||||
assert.deepEqual(await problemsOf(healed), []);
|
||||
});
|
||||
|
||||
test("a job told not to, or one that already alerts on every warning run, does not say it twice; a failed or silenced job is left to its own alerts", async () => {
|
||||
assert.deepEqual(await problemsOf(await jobWith("Quiet", ["Warning", "Warning", "Warning"], undefined, { warnStreakAlert: false })), []);
|
||||
const every = await problemsOf(await jobWith("Every", ["Warning", "Warning", "Warning"], undefined, { alertWarnings: true }));
|
||||
assert.deepEqual(every.map((x) => x.kind), ["warnings"], "the one that says it each time");
|
||||
assert.deepEqual(await problemsOf(await jobWith("Muted", ["Warning", "Warning", "Warning"], undefined, { alert: false })), []);
|
||||
});
|
||||
|
||||
test("a destination with a known size that fills within the window is an alert; further off, unknown, or erratic is not", async () => {
|
||||
// 20 days at 100 MB a day from 1 GB: now 2.9 GB of 3.4 GB, five days to full
|
||||
const soon = await jobWith("Soon", Array(20).fill("Success"), { start: 1e9, perDay: 100e6, quota: 3.4e9 });
|
||||
const p = await problemsOf(soon);
|
||||
assert.deepEqual(p.map((x) => [x.kind, x.severity]), [["growth", "warning"]]);
|
||||
assert.match(p[0].message, /“Soon” on Jackie is growing 100 MB a day and its storage will be full in about 5 days/);
|
||||
|
||||
const urgent = await jobWith("Urgent", Array(20).fill("Success"), { start: 1e9, perDay: 100e6, quota: 2.95e9 });
|
||||
assert.equal((await problemsOf(urgent))[0].severity, "critical", "three days or less");
|
||||
|
||||
assert.deepEqual(await problemsOf(soon, 3), [], "five days is outside a three day window");
|
||||
assert.deepEqual(await problemsOf(soon, 0), [], "switched off with zero");
|
||||
assert.deepEqual(await problemsOf(await jobWith("Far", Array(20).fill("Success"), { start: 1e9, perDay: 100e6, quota: 50e9 })), []);
|
||||
assert.deepEqual(await problemsOf(await jobWith("NoQuota", Array(20).fill("Success"), { start: 1e9, perDay: 100e6, quota: null })), [], "nothing to fill when it does not say how much room there is");
|
||||
});
|
||||
|
||||
test("the alerts belong to the server the machine runs on, so maintenance there silences them, and they are backup alerts about this integration", () => {
|
||||
assert.equal(subjectOfConditionKey("duplicati-streak:4:server:9"), "server:9");
|
||||
assert.equal(subjectOfConditionKey("duplicati-growth:4:server:9"), "server:9");
|
||||
assert.equal(subjectOfConditionKey("duplicati-growth:4"), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the page's data", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
let sourceId = 0;
|
||||
let jobId = 0;
|
||||
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "operator", name: "Olle", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vera", role: "viewer" },
|
||||
]);
|
||||
const made = await d.createSource({ name: "Route machine", serverId: null, createdBy: "Ada" });
|
||||
sourceId = made.row.id;
|
||||
const [b] = await db.insert(duplicatiBackups).values({ sourceId, name: "Routed", lastResult: "Warning", lastSuccessAt: new Date().toISOString(), warnings: 1, quotaBytes: 4e9 }).returning();
|
||||
jobId = b.id;
|
||||
const now = Date.now();
|
||||
await db.insert(duplicatiRuns).values(Array.from({ length: 20 }, (_, i) => ({ backupId: b.id, at: new Date(now - (19 - i + 0.2) * DAY).toISOString(), result: i >= 17 ? "Warning" : "Success", destinationBytes: 1e9 + i * 100e6 })));
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/duplicati", duplicatiRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/duplicati`;
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
test("each job says how many warnings in a row and how it is growing; the page is told the threshold and the window", async () => {
|
||||
who = "viewer";
|
||||
const body = await (await call("GET", "/")).json();
|
||||
const job = body.backups.find((b: { id: number }) => b.id === jobId);
|
||||
assert.equal(job.warningStreak, 3);
|
||||
assert.equal(job.growth.perDay, 100_000_000);
|
||||
assert.equal(job.growth.fit, 1);
|
||||
assert.equal(job.growth.daysToFull, Math.ceil((4e9 - 2.9e9) / 100e6));
|
||||
assert.equal(job.warnStreakAlert, true);
|
||||
assert.equal(job.destinationBytes, null, "this job's last report was not one that said");
|
||||
assert.equal(body.streakRuns, 3);
|
||||
assert.equal(body.forecastDays, 14);
|
||||
assert.match(job.problem, /warnings 3 runs in a row|growing 100 MB a day/);
|
||||
who = "operator";
|
||||
});
|
||||
|
||||
test("an operator can switch the streak alert off and on; a viewer cannot; the history carries the size", async () => {
|
||||
assert.equal((await call("PATCH", `/backups/${jobId}`, { warnStreakAlert: false })).status, 200);
|
||||
assert.equal((await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, jobId)))[0].warnStreakAlert, false);
|
||||
assert.equal((await call("PATCH", `/backups/${jobId}`, { warnStreakAlert: true })).status, 200);
|
||||
assert.equal((await call("PATCH", `/backups/${jobId}`, { warnStreakAlert: "yes" })).status, 400);
|
||||
who = "viewer";
|
||||
assert.equal((await call("PATCH", `/backups/${jobId}`, { warnStreakAlert: false })).status, 403);
|
||||
const runs = await (await call("GET", `/backups/${jobId}/runs?limit=3`)).json();
|
||||
assert.equal(runs.runs.length, 3);
|
||||
assert.equal(runs.runs[0].destinationBytes, 1e9 + 19 * 100e6);
|
||||
who = "operator";
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user