/** * Duplicati's reports: reading them (JSON in both shapes, and the plain text), what a report does to a backup and its history, where a backup stands, what is an alert, and the * address Duplicati posts to and the pages around it. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Server } from "node:http"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-duplicati-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers, duplicatiSources, duplicatiBackups, duplicatiRuns, auditLog } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const d = await import("../src/services/duplicati.js"); const { duplicatiReportRouter, duplicatiRouter } = await import("../src/routes/duplicati.js"); const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js"); const { subjectOfConditionKey, startOrExtendWindow } = await import("../src/services/maintenance.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { buildMetrics } = await import("../src/services/metrics.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); const HOUR = 3_600_000; const DAY = 86_400_000; /** A report as Duplicati sends it with the output format set to Json. */ const report = (over: Record = {}, name = "Documents", extra: Record = {}) => JSON.stringify({ Data: { DeletedFiles: 1, ModifiedFiles: 3, ExaminedFiles: 1520, AddedFiles: 2, SizeOfExaminedFiles: 3221225472, MainOperation: "Backup", ParsedResult: "Success", EndTime: "2026-10-23T03:35:44.1Z", BeginTime: "2026-10-23T03:30:31.4Z", Duration: "00:05:12.7", WarningsActualLength: 0, ErrorsActualLength: 0, Warnings: [], Errors: [], ...over }, Extra: { OperationName: "Backup", "machine-id": "abc", "backup-name": name, "backup-id": "DB-1", ...extra }, }); describe("reading a report", () => { test("a duration, in either of Duplicati's forms", () => { assert.equal(d.parseDuration("00:05:12.7"), 312); assert.equal(d.parseDuration("1.02:03:04"), 86_400 + 2 * 3600 + 3 * 60 + 4); assert.equal(d.parseDuration("12:00:00"), 43_200); for (const bad of [null, "", "5 minutes", 312, undefined]) assert.equal(d.parseDuration(bad), null); }); test("JSON with the results under Data and the job's name among Extra", () => { const r = d.parseReport(report())!; assert.deepEqual([r.operation, r.result, r.backupName, r.durationSeconds, r.examinedFiles, r.sizeBytes, r.addedFiles, r.modifiedFiles, r.deletedFiles, r.warnings, r.errors, r.message], ["Backup", "Success", "Documents", 312, 1520, 3221225472, 2, 3, 1, 0, 0, null]); assert.deepEqual([r.begin, r.end], ["2026-10-23T03:30:31.400Z", "2026-10-23T03:35:44.100Z"]); }); test("JSON with the results at the top, and a job known only by its id or by nothing", () => { const top = d.parseReport({ MainOperation: "Backup", ParsedResult: "Warning", WarningsActualLength: 2, Warnings: ["file locked", "another"], Extra: { "backup-name": "Photos" } })!; assert.deepEqual([top.backupName, top.result, top.warnings, top.message], ["Photos", "Warning", 2, "file locked"]); assert.equal(d.parseReport(report({}, "", { "backup-name": "", "backup-id": "DB-7" }))!.backupName, "Backup DB-7"); assert.equal(d.parseReport(JSON.stringify({ ParsedResult: "Success" }))!.backupName, "Unnamed backup"); }); test("a failure says why, with the first error", () => { const r = d.parseReport(report({ ParsedResult: "Fatal", ErrorsActualLength: 1, Errors: ["The backup storage destination is missing.\nIt may be offline"] }))!; assert.deepEqual([r.result, r.errors, r.message], ["Fatal", 1, "The backup storage destination is missing. It may be offline"]); }); test("the plain text Duplicati sends by default", () => { const text = ["DeletedFiles: 0", "ModifiedFiles: 3", "ExaminedFiles: 1,520", "SizeOfExaminedFiles: 3221225472 (3.00 GB)", "MainOperation: Backup", "ParsedResult: Success", "Duration: 00:05:12.7", "WarningsActualLength: 0", "backup-name: Documents"].join("\n"); const r = d.parseReport(text)!; assert.deepEqual([r.operation, r.result, r.backupName, r.durationSeconds, r.examinedFiles, r.sizeBytes, r.modifiedFiles], ["Backup", "Success", "Documents", 312, 1520, 3221225472, 3]); }); test("what is not a report is not read", () => { for (const nothing of ["", "hello", "{}", "{not json", JSON.stringify({ Data: { MainOperation: "Backup" } }), null, 5, []]) assert.equal(d.parseReport(nothing as never), null, String(nothing)); assert.equal(d.parseReport(report({ MainOperation: "Compact" }))?.operation, "Compact"); }); }); describe("what a report does", () => { let token = ""; let sourceId = 0; before(async () => { const made = await d.createSource({ name: "Jackie", serverId: null, createdBy: "Ada" }); token = made.token; sourceId = made.row.id; }); const backup = async (name = "Documents") => (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, name)))[0]; test("the secret is shown once and only its hash is kept", async () => { assert.match(token, /^hlm_d_[\w-]{20,}$/); const [row] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, sourceId)); assert.equal(row.tokenPrefix, token.slice(0, 12)); assert.ok(!JSON.stringify(row).includes(token), "the token itself is nowhere in the row"); assert.equal((await d.sourceForToken(token))?.id, sourceId); for (const bad of ["", "short", `${token}x`, "x".repeat(200)]) assert.equal(await d.sourceForToken(bad), null); }); test("a first report teaches the app the job; what it says is kept", async () => { const t0 = new Date("2026-10-23T04:00:00Z"); assert.deepEqual(await d.recordReport(token, report(), t0), { recorded: true, backupId: (await backup())?.id ?? 1, result: "Success" }); const b = await backup(); assert.deepEqual([b.name, b.lastResult, b.durationSeconds, b.sizeBytes, b.addedFiles, b.modifiedFiles, b.deletedFiles, b.consecutiveFailures, b.expectedHours, b.alert], ["Documents", "Success", 312, 3221225472, 2, 3, 1, 0, 48, true]); assert.equal(b.lastSuccessAt, "2026-10-23T03:35:44.100Z", "when the backup finished, from its own report"); const [src] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, sourceId)); assert.equal(src.lastReportAt, t0.toISOString()); const runs = await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, b.id)); assert.deepEqual(runs.map((r) => [r.result, r.durationSeconds, r.filesChanged]), [["Success", 312, 6]]); }); test("failures are counted in a row, the last good one is kept, and a good run (warnings included) ends the run of failures", async () => { const fail = (end: string) => report({ ParsedResult: "Error", ErrorsActualLength: 1, Errors: ["Unable to connect to the backend"], EndTime: end }); await d.recordReport(token, fail("2026-10-24T03:35:00Z"), new Date("2026-10-24T03:36:00Z")); let b = await backup(); assert.deepEqual([b.lastResult, b.consecutiveFailures, b.lastSuccessAt, b.message], ["Error", 1, "2026-10-23T03:35:44.100Z", "Unable to connect to the backend"]); await d.recordReport(token, fail("2026-10-25T03:35:00Z"), new Date("2026-10-25T03:36:00Z")); assert.equal((await backup()).consecutiveFailures, 2); await d.recordReport(token, report({ ParsedResult: "Warning", WarningsActualLength: 1, Warnings: ["locked file"], EndTime: "2026-10-26T03:35:00Z" }), new Date("2026-10-26T03:36:00Z")); b = await backup(); assert.deepEqual([b.lastResult, b.consecutiveFailures, b.lastSuccessAt, b.warnings], ["Warning", 0, "2026-10-26T03:35:00.000Z", 1]); }); test("a compact, a repair or a test is set aside; an unreadable report and an unknown address are told apart", async () => { const before = (await db.select().from(duplicatiRuns)).length; assert.deepEqual(await d.recordReport(token, report({ MainOperation: "Compact" })), { recorded: false, reason: "not_a_backup" }); assert.deepEqual(await d.recordReport(token, "nonsense"), { recorded: false, reason: "unreadable" }); assert.deepEqual(await d.recordReport("hlm_d_not-a-real-token-at-all", report()), { recorded: false, reason: "unknown_token" }); assert.equal((await db.select().from(duplicatiRuns)).length, before, "nothing was written down"); }); test("two jobs on one machine are two backups; the same name on another machine is another", async () => { await d.recordReport(token, report({}, "Photos"), new Date("2026-10-26T04:00:00Z")); const other = await d.createSource({ name: "Nova", serverId: null, createdBy: "Ada" }); await d.recordReport(other.token, report({}, "Documents"), new Date("2026-10-26T04:00:00Z")); const all = await db.select().from(duplicatiBackups); assert.deepEqual(all.map((b) => `${b.sourceId}:${b.name}`).sort(), [`${sourceId}:Documents`, `${sourceId}:Photos`, `${other.row.id}:Documents`].sort()); }); test("a job's history is kept at most 400 reports long, and none older than 400 days", async () => { const b = await backup("Photos"); const base = Date.parse("2026-10-26T00:00:00Z"); for (let i = 0; i < 405; i++) await db.insert(duplicatiRuns).values({ backupId: b.id, at: new Date(base - (i + 1) * HOUR).toISOString(), result: "Success", warnings: 0, errors: 0 }); await db.insert(duplicatiRuns).values({ backupId: b.id, at: new Date(base - 500 * DAY).toISOString(), result: "Success", warnings: 0, errors: 0 }); await d.recordReport(token, report({ EndTime: "2026-10-26T05:00:00Z" }, "Photos"), new Date("2026-10-26T05:01:00Z")); const runs = await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, b.id)); assert.equal(runs.length, 400); assert.ok(runs.every((r) => Date.parse(r.at) > base - 400 * DAY)); }); }); describe("where a backup stands, and what is an alert", () => { const row = (extra: Partial = {}): import("../src/services/duplicati.js").BackupRow => ({ id: 1, sourceId: 1, name: "Documents", lastReportAt: null, lastResult: "Success", lastBegin: null, lastEnd: null, durationSeconds: null, examinedFiles: null, sizeBytes: null, addedFiles: null, modifiedFiles: null, deletedFiles: null, warnings: 0, errors: 0, message: null, lastSuccessAt: new Date(NOW - 5 * HOUR).toISOString(), consecutiveFailures: 0, expectedHours: 48, alert: true, createdAt: "2026-10-01 00:00:00", ...extra, }); const NOW = Date.parse("2026-10-24T12:00:00Z"); const iso = (hoursAgo: number) => new Date(NOW - hoursAgo * HOUR).toISOString(); test("ok, warning, failed and stale", () => { assert.equal(d.backupState(row(), NOW), "ok"); assert.equal(d.backupState(row({ lastResult: "Warning" }), NOW), "warning"); assert.equal(d.backupState(row({ lastResult: "Error", consecutiveFailures: 1 }), NOW), "failed"); assert.equal(d.backupState(row({ lastResult: "Fatal" }), NOW), "failed"); assert.equal(d.backupState(row({ lastSuccessAt: iso(47) }), NOW), "ok", "inside what is expected"); assert.equal(d.backupState(row({ lastSuccessAt: iso(49) }), NOW), "stale"); assert.equal(d.backupState(row({ lastSuccessAt: iso(30), expectedHours: 24 }), NOW), "stale", "it is per job"); assert.equal(d.backupState(row({ lastResult: "Warning", lastSuccessAt: iso(60) }), NOW), "stale", "overdue beats warnings"); }); test("a failure is a warning, critical from the second in a row; a stale one is critical at three times what is expected", () => { const sources = [{ id: 1, name: "Jackie", serverId: null }]; const p = (b: Partial) => d.duplicatiProblems(sources, [row(b)], NOW); assert.deepEqual(p({}), []); const first = p({ lastResult: "Error", consecutiveFailures: 1, message: "Unable to connect to the backend", lastSuccessAt: iso(30) })[0]; assert.deepEqual([first.key, first.kind, first.severity], ["duplicati-failed:1", "failed", "warning"]); assert.equal(first.message, "The backup “Documents” on Jackie failed: Unable to connect to the backend (the last good one was 30 h ago)"); assert.equal(p({ lastResult: "Error", consecutiveFailures: 2 })[0].severity, "critical"); assert.match(p({ lastResult: "Fatal", consecutiveFailures: 1, lastSuccessAt: null })[0].message, /there has been no good one yet/); const stale = p({ lastSuccessAt: iso(72) })[0]; assert.deepEqual([stale.key, stale.severity], ["duplicati-stale:1", "warning"]); assert.equal(stale.message, "The backup “Documents” on Jackie hasn't succeeded for 3 days — it should run at least every 2 days"); assert.equal(p({ lastSuccessAt: iso(150) })[0].severity, "critical"); assert.match(p({ lastSuccessAt: iso(30), expectedHours: 24 })[0].message, /every 24 h/); }); test("one with its alert switched off says nothing; the server a machine runs on is in the key, so its maintenance can silence it", () => { assert.deepEqual(d.duplicatiProblems([{ id: 1, name: "Jackie", serverId: null }], [row({ lastResult: "Error", alert: false })], NOW), []); const [p] = d.duplicatiProblems([{ id: 1, name: "Jackie", serverId: 9 }], [row({ lastResult: "Error", consecutiveFailures: 1 })], NOW); assert.equal(p.key, "duplicati-failed:1:server:9"); assert.equal(subjectOfConditionKey(p.key), "server:9"); assert.equal(subjectOfConditionKey("duplicati-stale:4:server:2"), "server:2"); assert.equal(subjectOfConditionKey("duplicati-failed:1"), null, "a machine that is not tied to a server is not silenced by one"); }); }); describe("the address Duplicati posts to, the pages, the alerts and /metrics", () => { let app: Server; let base = ""; let who = "admin"; let token = ""; let serverId = 0; before(async () => { // what the earlier groups made is not this group's: a clean start (a machine takes its backups and their history with it) await db.delete(duplicatiSources); await db.insert(users).values([ { oidcSub: "admin", name: "Ada", role: "admin" }, { oidcSub: "operator", name: "Olga", role: "operator" }, { oidcSub: "viewer", name: "Vic", role: "viewer" }, ]); serverId = (await db.insert(servers).values({ name: "Jackie", apiTokenHash: "j", apiTokenPrefix: "j", lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id; const a = express(); // the address Duplicati posts to has no session and is mounted first, as in the app a.use("/api/duplicati-report", duplicatiReportRouter); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/duplicati", duplicatiRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); const call = (method: string, path: string, body?: unknown, headers: Record = {}) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json", connection: "close", ...headers }, body: body === undefined ? undefined : typeof body === "string" ? body : JSON.stringify(body) }); test("an administrator makes a machine and is shown its secret address once; nobody else can, and it is in the audit log", async () => { who = "operator"; assert.equal((await call("POST", "/duplicati/sources", { name: "Nope" })).status, 403); who = "admin"; for (const bad of [{}, { name: " " }, { name: "x", serverId: 99999 }]) assert.equal((await call("POST", "/duplicati/sources", bad)).status, 400, JSON.stringify(bad)); const res = await call("POST", "/duplicati/sources", { name: "Jackie", serverId }); assert.equal(res.status, 201); const made = (await res.json()) as { source: Record; token: string; path: string }; token = made.token; assert.equal(made.path, `/api/duplicati-report/${token}`); assert.ok(!("tokenHash" in made.source)); const list = JSON.stringify(await (await call("GET", "/duplicati")).json()); assert.ok(!list.includes(token) && !list.includes("tokenHash"), "never shown again"); assert.equal((await db.select().from(auditLog).where(eq(auditLog.category, "duplicati"))).length, 1); }); test("Duplicati posts to the address with no sign-in: JSON or text; a wrong address is not found, and a GET is refused", async () => { const url = `/duplicati-report/${token}`; const good = await call("POST", url, report({ EndTime: new Date().toISOString() }, "Documents"), { "content-type": "text/plain" }); assert.deepEqual([good.status, await good.text()], [200, "OK"]); const text = await call("POST", url, ["MainOperation: Backup", "ParsedResult: Success", "Duration: 00:01:00", "backup-name: Photos"].join("\n"), { "content-type": "text/plain" }); assert.equal(text.status, 200); const compact = await call("POST", url, report({ MainOperation: "Compact" }), { "content-type": "text/plain" }); assert.match(await compact.text(), /not a backup/); assert.match(await (await call("POST", url, "rubbish", { "content-type": "text/plain" })).text(), /not a report this app can read/); assert.equal((await call("POST", "/duplicati-report/hlm_d_this-token-does-not-exist-1234", report())).status, 404); assert.equal((await call("GET", url)).status, 405); assert.equal((await db.select().from(duplicatiBackups)).length, 2, "Documents and Photos"); }); test("anyone signed in sees each backup, where it stands and what is wrong; the history of one is there too", async () => { who = "viewer"; await d.recordReport(token, report({ ParsedResult: "Fatal", ErrorsActualLength: 1, Errors: ["destination is missing"], EndTime: new Date().toISOString() }, "Photos")); const body = (await (await call("GET", "/duplicati")).json()) as any; assert.deepEqual(body.sources.map((s: any) => [s.name, s.serverName, s.backups]), [["Jackie", "Jackie", 2]]); const photos = body.backups.find((b: any) => b.name === "Photos"); assert.deepEqual([photos.state, photos.sourceName, /destination is missing/.test(photos.problem)], ["failed", "Jackie", true]); assert.equal(body.backups.find((b: any) => b.name === "Documents").state, "ok"); const runs = (await (await call("GET", `/duplicati/backups/${photos.id}/runs`)).json()) as any; assert.deepEqual([runs.backup.name, runs.runs.length, runs.runs[0].result], ["Photos", 2, "Fatal"]); assert.equal((await call("GET", "/duplicati/backups/999/runs")).status, 404); who = "admin"; }); test("a failed backup is an alert in the Backups kind, with where to look; maintenance on its server silences it", async () => { invalidateAlertsCache(); const report1 = await getAlerts(true); const a = report1.alerts.find((x) => /^backup:duplicati-failed:\d+:server:/.test(x.id))!; assert.ok(a, "the alert is there"); assert.deepEqual([a.severity, a.category, a.source, a.link, a.silenced], ["warning", "backup", "Duplicati", "/duplicati", false]); assert.match(a.message, /^The backup “Photos” on Jackie failed: destination is missing/); await startOrExtendWindow("server", serverId, 60, "test", "t"); invalidateAlertsCache(); assert.equal((await getAlerts(true)).alerts.find((x) => x.id === a.id)?.silenced, true); const { snapshot } = await collectSnapshot(); assert.ok(snapshot.duplicati?.some((p) => p.kind === "failed"), "the health pass reads it too"); const list = evaluateHealth({ servers: [], proxmox: [], synology: [], duplicati: snapshot.duplicati }, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now()); assert.ok(list.some((c) => c.source === "duplicati" && c.key.startsWith("duplicati-failed:"))); }); test("an operator sets how often a backup is expected and whether it alerts; a viewer cannot, and a wrong number is refused", async () => { const photos = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Photos")))[0]; who = "viewer"; assert.equal((await call("PATCH", `/duplicati/backups/${photos.id}`, { expectedHours: 24 })).status, 403); who = "operator"; assert.equal((await call("PATCH", `/duplicati/backups/${photos.id}`, { expectedHours: 168, alert: false })).status, 200); const after = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, photos.id)))[0]; assert.deepEqual([after.expectedHours, after.alert], [168, false]); for (const bad of [{ expectedHours: 0 }, { expectedHours: 5000 }, { expectedHours: 1.5 }]) assert.equal((await call("PATCH", `/duplicati/backups/${photos.id}`, bad)).status, 400, JSON.stringify(bad)); assert.equal((await call("PATCH", "/duplicati/backups/999", { alert: true })).status, 404); invalidateAlertsCache(); assert.ok(!(await getAlerts(true)).alerts.some((x) => x.id.includes("duplicati-failed")), "no alert with it switched off"); await call("PATCH", `/duplicati/backups/${photos.id}`, { alert: true }); who = "admin"; }); test("/metrics has each backup: in order or not, how long since it last worked, and how big it is", async () => { const text = await buildMetrics(); assert.match(text, /homelab_duplicati_backup_ok\{backup="Documents",source="Jackie"\} 1/); assert.match(text, /homelab_duplicati_backup_ok\{backup="Photos",source="Jackie"\} 0/); assert.match(text, /homelab_duplicati_last_success_age_seconds\{backup="Documents",source="Jackie"\} \d+/); assert.match(text, /homelab_duplicati_source_size_bytes\{backup="Documents",source="Jackie"\} 3221225472/); }); test("a backup can be forgotten, and a machine removed with all of them; both are for people who may, and in the audit log", async () => { const photos = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Photos")))[0]; who = "viewer"; assert.equal((await call("DELETE", `/duplicati/backups/${photos.id}`)).status, 403); who = "operator"; assert.equal((await call("DELETE", `/duplicati/backups/${photos.id}`)).status, 204); assert.equal((await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, photos.id))).length, 0, "its history goes with it"); assert.equal((await call("DELETE", `/duplicati/sources/${(await db.select().from(duplicatiSources))[0].id}`)).status, 403, "removing a machine is an administrator's"); who = "admin"; const [src] = await db.select().from(duplicatiSources); assert.equal((await call("PATCH", `/duplicati/sources/${src.id}`, { name: "Jackie (docker)", serverId })).status, 200); assert.equal((await call("DELETE", `/duplicati/sources/${src.id}`)).status, 204); assert.equal((await db.select().from(duplicatiBackups)).length, 0, "its backups go with it"); assert.equal((await call("POST", `/duplicati-report/${token}`, report(), { "content-type": "text/plain" })).status, 404, "and its address stops working"); const actions = (await db.select().from(auditLog).where(eq(auditLog.category, "duplicati"))).map((a) => a.action); assert.deepEqual(actions.sort(), ["create_source", "delete_source", "forget_backup", "update_backup", "update_backup", "update_source"].sort()); }); });