/** * Duplicati, the slower things: a job whose last runs in a row all had warnings, and how much room the backups take where they are kept (read from what Duplicati reports, worked into a growth and a * forecast, and an alert when a destination with a known size will be full within the forecast window). * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Server } from "node:http"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-dupfollow-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, duplicatiSources, duplicatiBackups, duplicatiRuns } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const d = await import("../src/services/duplicati.js"); const { duplicatiRouter } = await import("../src/routes/duplicati.js"); const { subjectOfConditionKey } = await import("../src/services/maintenance.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); const DAY = 86_400_000; const NOW = new Date("2026-11-10T12:00:00.000Z"); const daysAgo = (n: number) => new Date(NOW.getTime() - n * DAY).toISOString(); const report = (over: Record = {}, backend: Record | null = { KnownFileSize: 5_000_000_000, TotalQuotaSpace: 0 }, name = "Documents") => JSON.stringify({ Data: { MainOperation: "Backup", ParsedResult: "Success", EndTime: "2026-10-23T03:35:44.1Z", BeginTime: "2026-10-23T03:30:31.4Z", Duration: "00:05:12.7", Warnings: [], Errors: [], ...(backend ? { BackendStatistics: backend } : {}), ...over }, Extra: { OperationName: "Backup", "backup-name": name }, }); describe("what the report says about the destination", () => { test("JSON: the size where the backups are kept, and the room there is when its storage says", () => { const r = d.parseReport(report({}, { KnownFileSize: 12_345_678, TotalQuotaSpace: 500_000_000_000, FreeQuotaSpace: 1 }))!; assert.deepEqual([r.destinationBytes, r.quotaBytes], [12_345_678, 500_000_000_000]); }); test("a quota of zero or less is not said, a size below zero is not said, and a report without the section says neither", () => { assert.deepEqual(d.destinationOf(100, 0), { destinationBytes: 100, quotaBytes: null }); assert.deepEqual(d.destinationOf(100, -1), { destinationBytes: 100, quotaBytes: null }); assert.deepEqual(d.destinationOf(-5, 10), { destinationBytes: null, quotaBytes: 10 }); assert.deepEqual(d.destinationOf(null, null), { destinationBytes: null, quotaBytes: null }); const none = d.parseReport(report({}, null))!; assert.deepEqual([none.destinationBytes, none.quotaBytes], [null, null]); const zero = d.parseReport(report({}, { KnownFileSize: 0, TotalQuotaSpace: 0 }))!; assert.deepEqual([zero.destinationBytes, zero.quotaBytes], [0, null], "an empty destination is a size, not a missing one"); }); test("the plain text form: the bytes come first, the readable size after", () => { const text = ["MainOperation: Backup", "ParsedResult: Success", "Duration: 00:01:00", "BackendStatistics:", " KnownFileSize: 5811709 (5.54 MB)", " TotalQuotaSpace: 107374182400 (100.00 GB)", " FreeQuotaSpace: 0", "backup-name: Docs"].join("\n"); const r = d.parseReport(text)!; assert.deepEqual([r.destinationBytes, r.quotaBytes], [5_811_709, 107_374_182_400]); }); }); describe("warnings in a row", () => { test("counted from the newest run; a success, a failure or an unknown word ends it", () => { assert.equal(d.warningStreak([]), 0); assert.equal(d.warningStreak(["Warning", "Warning", "Warning", "Success", "Warning"]), 3); assert.equal(d.warningStreak(["warning", "WARNING"]), 2); assert.equal(d.warningStreak(["Success", "Warning", "Warning"]), 0); assert.equal(d.warningStreak(["Warning", "Error", "Warning"]), 1); assert.equal(d.STREAK_RUNS, 3); }); }); describe("growth", () => { const steady = (days: number, perDay: number, start = 1e9, every = 1) => Array.from({ length: Math.floor(days / every) + 1 }, (_, i) => ({ t: NOW.getTime() - (days - i * every) * DAY, v: start + i * every * perDay })); test("a steady climb: bytes a day and a month, how well a line fits, and when a known quota is full", () => { const g = d.growthOf(steady(20, 50_000_000), 5e9)!; assert.equal(g.perDay, 50_000_000); assert.equal(g.perMonth, 1_500_000_000); assert.equal(g.fit, 1); assert.equal(g.spanDays, 20); assert.equal(g.latest, 2e9); assert.equal(g.daysToFull, 60, "(5 GB − 2 GB) at 50 MB a day"); }); test("no quota, nothing to fill; shrinking, nothing to fill; already over, nothing to fill", () => { assert.equal(d.growthOf(steady(20, 50_000_000), null)!.daysToFull, null); const shrinking = d.growthOf(steady(20, -20_000_000, 5e9), 9e9)!; assert.ok(shrinking.perDay < 0); assert.equal(shrinking.daysToFull, null); assert.equal(d.growthOf(steady(20, 50_000_000), 1e9)!.daysToFull, null); }); test("not enough to go on: too few readings, or too short a time", () => { assert.equal(d.growthOf(steady(20, 1, 1e9, 10), null), null, "three readings"); assert.equal(d.growthOf(steady(5, 1_000, 1e9, 1), null), null, "five days"); assert.equal(d.growthOf([], null), null); const flat = d.growthOf(steady(20, 0), 5e9)!; assert.deepEqual([flat.perDay, flat.fit, flat.daysToFull], [0, 0, null], "a size that does not move has no fit and nothing to fill"); }); test("a jagged size (a compaction now and then) fits badly", () => { const jagged = steady(30, 0).map((p, i) => ({ ...p, v: 1e9 + (i % 5 === 0 ? 0 : 3e8) * (i % 3) })); assert.ok(d.growthOf(jagged, 5e9)!.fit < d.GROWTH_MIN_FIT); }); }); describe("what a report does to the destination, and the alerts from it", () => { let sourceId = 0; let token = ""; before(async () => { const made = await d.createSource({ name: "Jackie", serverId: null, createdBy: "Ada" }); sourceId = made.row.id; token = made.token; }); after(() => { try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); const backupId = async (name: string) => (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, name)))[0].id; test("it is written on the job and on the run; a report that does not say leaves what was known", async () => { await d.recordReport(token, report({}, { KnownFileSize: 7_000_000_000, TotalQuotaSpace: 9_000_000_000 }, "Docs"), new Date("2026-11-01T04:00:00Z")); const id = await backupId("Docs"); let [b] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id)); assert.deepEqual([b.destinationBytes, b.quotaBytes], [7_000_000_000, 9_000_000_000]); assert.equal((await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, id)))[0].destinationBytes, 7_000_000_000); await d.recordReport(token, report({}, null, "Docs"), new Date("2026-11-02T04:00:00Z")); [b] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id)); assert.deepEqual([b.destinationBytes, b.quotaBytes], [7_000_000_000, 9_000_000_000], "kept"); const runs = await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, id)); assert.deepEqual(runs.map((r) => r.destinationBytes).sort(), [7_000_000_000, null], "the run itself says nothing"); }); /** A job with a history made directly: the results newest last, one a day, and (optionally) a growing size. */ async function jobWith(name: string, results: string[], size?: { start: number; perDay: number; quota: number | null }, flags: Partial = {}) { const [b] = await db .insert(duplicatiBackups) .values({ sourceId, name, lastResult: results[results.length - 1], lastSuccessAt: daysAgo(0.5), lastReportAt: daysAgo(0.5), warnings: results[results.length - 1] === "Warning" ? 2 : 0, warningMessages: JSON.stringify(["locked file", "another"]), quotaBytes: size?.quota ?? null, ...flags }) .returning(); await db.insert(duplicatiRuns).values(results.map((result, i) => ({ backupId: b.id, at: daysAgo(results.length - 1 - i + 0.5), result, warnings: result === "Warning" ? 1 : 0, errors: 0, destinationBytes: size ? size.start + i * size.perDay : null }))); return b; } const problemsOf = async (b: typeof duplicatiBackups.$inferSelect, forecastDays = 14) => { const [fresh] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, b.id)); const [source] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, sourceId)); const byBackup = await d.analyseBackups([fresh], NOW); return d.duplicatiProblems([source], [fresh], NOW.getTime(), { byBackup, forecastDays }).filter((p) => p.key.endsWith(`:${b.id}`) || p.key.includes(`:${b.id}:`)); }; test("three warnings in a row is said once, with what the last said; two is not, and a success after them ends it", async () => { const three = await jobWith("Three", ["Success", "Warning", "Warning", "Warning"]); const p = await problemsOf(three); assert.deepEqual(p.map((x) => [x.key, x.kind, x.severity]), [[`duplicati-streak:${three.id}`, "streak", "warning"]]); assert.match(p[0].message, /“Three” on Jackie has finished with warnings 3 runs in a row — the last said: locked file/); const two = await jobWith("Two", ["Success", "Warning", "Warning"]); assert.deepEqual(await problemsOf(two), []); const healed = await jobWith("Healed", ["Warning", "Warning", "Warning", "Success"]); assert.deepEqual(await problemsOf(healed), []); }); test("a job told not to, or one that already alerts on every warning run, does not say it twice; a failed or silenced job is left to its own alerts", async () => { assert.deepEqual(await problemsOf(await jobWith("Quiet", ["Warning", "Warning", "Warning"], undefined, { warnStreakAlert: false })), []); const every = await problemsOf(await jobWith("Every", ["Warning", "Warning", "Warning"], undefined, { alertWarnings: true })); assert.deepEqual(every.map((x) => x.kind), ["warnings"], "the one that says it each time"); assert.deepEqual(await problemsOf(await jobWith("Muted", ["Warning", "Warning", "Warning"], undefined, { alert: false })), []); }); test("a destination with a known size that fills within the window is an alert; further off, unknown, or erratic is not", async () => { // 20 days at 100 MB a day from 1 GB: now 2.9 GB of 3.4 GB, five days to full const soon = await jobWith("Soon", Array(20).fill("Success"), { start: 1e9, perDay: 100e6, quota: 3.4e9 }); const p = await problemsOf(soon); assert.deepEqual(p.map((x) => [x.kind, x.severity]), [["growth", "warning"]]); assert.match(p[0].message, /“Soon” on Jackie is growing 100 MB a day and its storage will be full in about 5 days/); const urgent = await jobWith("Urgent", Array(20).fill("Success"), { start: 1e9, perDay: 100e6, quota: 2.95e9 }); assert.equal((await problemsOf(urgent))[0].severity, "critical", "three days or less"); assert.deepEqual(await problemsOf(soon, 3), [], "five days is outside a three day window"); assert.deepEqual(await problemsOf(soon, 0), [], "switched off with zero"); assert.deepEqual(await problemsOf(await jobWith("Far", Array(20).fill("Success"), { start: 1e9, perDay: 100e6, quota: 50e9 })), []); assert.deepEqual(await problemsOf(await jobWith("NoQuota", Array(20).fill("Success"), { start: 1e9, perDay: 100e6, quota: null })), [], "nothing to fill when it does not say how much room there is"); }); test("the alerts belong to the server the machine runs on, so maintenance there silences them, and they are backup alerts about this integration", () => { assert.equal(subjectOfConditionKey("duplicati-streak:4:server:9"), "server:9"); assert.equal(subjectOfConditionKey("duplicati-growth:4:server:9"), "server:9"); assert.equal(subjectOfConditionKey("duplicati-growth:4"), null); }); }); describe("the page's data", () => { let app: Server; let base = ""; let who = "operator"; let sourceId = 0; let jobId = 0; const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); before(async () => { await db.insert(users).values([ { oidcSub: "operator", name: "Olle", role: "operator" }, { oidcSub: "viewer", name: "Vera", role: "viewer" }, ]); const made = await d.createSource({ name: "Route machine", serverId: null, createdBy: "Ada" }); sourceId = made.row.id; const [b] = await db.insert(duplicatiBackups).values({ sourceId, name: "Routed", lastResult: "Warning", lastSuccessAt: new Date().toISOString(), warnings: 1, quotaBytes: 4e9 }).returning(); jobId = b.id; const now = Date.now(); await db.insert(duplicatiRuns).values(Array.from({ length: 20 }, (_, i) => ({ backupId: b.id, at: new Date(now - (19 - i + 0.2) * DAY).toISOString(), result: i >= 17 ? "Warning" : "Success", destinationBytes: 1e9 + i * 100e6 }))); const a = express(); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/duplicati", duplicatiRouter); app = await new Promise((resolve) => { const x = a.listen(0, "127.0.0.1", () => resolve(x)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/duplicati`; }); after(() => app.close()); test("each job says how many warnings in a row and how it is growing; the page is told the threshold and the window", async () => { who = "viewer"; const body = await (await call("GET", "/")).json(); const job = body.backups.find((b: { id: number }) => b.id === jobId); assert.equal(job.warningStreak, 3); assert.equal(job.growth.perDay, 100_000_000); assert.equal(job.growth.fit, 1); assert.equal(job.growth.daysToFull, Math.ceil((4e9 - 2.9e9) / 100e6)); assert.equal(job.warnStreakAlert, true); assert.equal(job.destinationBytes, null, "this job's last report was not one that said"); assert.equal(body.streakRuns, 3); assert.equal(body.forecastDays, 14); assert.match(job.problem, /warnings 3 runs in a row|growing 100 MB a day/); who = "operator"; }); test("an operator can switch the streak alert off and on; a viewer cannot; the history carries the size", async () => { assert.equal((await call("PATCH", `/backups/${jobId}`, { warnStreakAlert: false })).status, 200); assert.equal((await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, jobId)))[0].warnStreakAlert, false); assert.equal((await call("PATCH", `/backups/${jobId}`, { warnStreakAlert: true })).status, 200); assert.equal((await call("PATCH", `/backups/${jobId}`, { warnStreakAlert: "yes" })).status, 400); who = "viewer"; assert.equal((await call("PATCH", `/backups/${jobId}`, { warnStreakAlert: false })).status, 403); const runs = await (await call("GET", `/backups/${jobId}/runs?limit=3`)).json(); assert.equal(runs.runs.length, 3); assert.equal(runs.runs[0].destinationBytes, 1e9 + 19 * 100e6); who = "operator"; }); });