/** * Restore tests: what the newest test of something amounts to, when to speak up, and the routes that record them. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; import type { Server } from "node:http"; const dir = mkdtempSync(join(tmpdir(), "hm-restore-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, restoreTests, auditLog } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { restoreInfo, restoreAlerts, customKey, guestKey } = await import("../src/services/restoreTests.js"); const { setCoverageLoaderForTests, buildCoverage } = await import("../src/services/backupCoverage.js"); const { backupsRouter } = await import("../src/routes/backups.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { updateSettings } = await import("../src/services/settingsStore.js"); const { buildMetrics } = await import("../src/services/metrics.js"); await runMigrations(); const NOW = new Date("2026-10-18T12:00:00Z"); const daysAgo = (n: number) => new Date(NOW.getTime() - n * 86_400_000).toISOString().slice(0, 10); const test1 = (result: string, ago: number, extra: Record = {}) => ({ result, testedAt: daysAgo(ago), note: null as string | null, testedBy: "Ada", ...extra }); describe("what the newest test amounts to", () => { test("how it went; none at all is never; an old pass is stale once there is a limit; a failure or 'not needed' doesn't age", () => { assert.equal(restoreInfo(undefined, NOW, 90).status, "never"); assert.deepEqual(restoreInfo(test1("ok", 10), NOW, 90), { status: "ok", testedAt: daysAgo(10), ageDays: 10, note: null, testedBy: "Ada" }); assert.equal(restoreInfo(test1("ok", 100), NOW, 90).status, "stale"); assert.equal(restoreInfo(test1("ok", 100), NOW, 0).status, "ok", "no limit, no staleness"); assert.equal(restoreInfo(test1("partial", 100), NOW, 90).status, "stale"); assert.equal(restoreInfo(test1("failed", 400), NOW, 90).status, "failed"); assert.equal(restoreInfo(test1("not_needed", 400), NOW, 90).status, "not_needed"); }); test("something that isn't a guest is told apart by its name", () => { assert.equal(customKey("Photos share"), "custom:photos-share"); assert.equal(customKey(" Gitea — database! "), "custom:gitea-database"); assert.equal(customKey("Åkerblom's NAS"), "custom:akerblom-s-nas"); assert.equal(customKey("!!!"), null); assert.equal(guestKey(2, 105), "guest:2:105"); }); }); const guest = (vmid: number, name: string) => ({ vmid, name, node: "pve1", type: "qemu" as const, status: "running", cpu: null, maxmem: null, mem: null, uptime: null }); const fixture = () => { const loaded = { proxmox: [{ integrationId: 1, integrationName: "Felicia", guests: [guest(100, "elsa"), guest(101, "anna"), guest(102, "zoe")], jobs: [{ id: "j", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null }], tasks: [] }], pbs: [{ integrationName: "Backup server", groups: [{ backupType: "vm", backupId: "100", backupTime: Math.floor(Date.now() / 1000) - 3600, verification: "ok" as const }, { backupType: "vm", backupId: "101", backupTime: Math.floor(Date.now() / 1000) - 7200, verification: "ok" as const }] }], sources: [{ kind: "proxmox" as const, name: "Felicia", ok: true }], }; return loaded; }; describe("when to speak up", () => { const report = () => ({ guests: buildCoverage({ ...fixture(), servers: [], now: NOW }), counts: {} as never, sources: [], staleHours: 48, generatedAt: NOW.toISOString() }); const row = (key: string, label: string, result: string, ago: number) => [key, { id: 1, subjectKey: key, label, testedAt: daysAgo(ago), result, note: null, testedBy: null, createdAt: "" }] as const; test("with no limit only a failure speaks; with one, a guest that has backups and no recent test does too", () => { const g = report(); assert.deepEqual(restoreAlerts(g, new Map(), 0, NOW), [], "no limit, nothing tested: quiet"); const limited = restoreAlerts(g, new Map(), 90, NOW); assert.deepEqual(limited.map((a) => [a.guest?.name, a.severity]), [["elsa", "warning"], ["anna", "warning"]], "zoe has no backups to restore from, so nothing to say"); assert.match(limited[0].message, /elsa \(#100\) \[Felicia\]: a restore has never been tested/); const failed = restoreAlerts(g, new Map([row("guest:1:100", "elsa", "failed", 3)]), 0, NOW); assert.equal(failed.length, 1); assert.equal(failed[0].severity, "critical"); assert.match(failed[0].message, /the last restore test \(.*\) failed/); }); test("a recent pass, 'not needed' and an old pass under the limit are all fine; an old one isn't", () => { const g = report(); const tests = new Map([row("guest:1:100", "elsa", "ok", 10), row("guest:1:101", "anna", "not_needed", 500)]); assert.deepEqual(restoreAlerts(g, tests, 90, NOW), []); const old = restoreAlerts(g, new Map([row("guest:1:100", "elsa", "ok", 120), row("guest:1:101", "anna", "ok", 1)]), 90, NOW); assert.equal(old.length, 1); assert.match(old[0].message, /last restore test was 120 days ago \(the limit is 90 days\)/); }); test("something that isn't a guest is told of when it failed, or when it is stale", () => { const g = report(); const tests = new Map([row("custom:photos", "Photos share", "ok", 200), row("custom:db", "Gitea database", "failed", 5), row("custom:fine", "Fine", "ok", 3), row("guest:9:9", "gone guest", "ok", 999)]); const out = restoreAlerts(g, tests, 90, NOW); assert.deepEqual(out.map((a) => a.message.split(":")[0]).sort(), ["Gitea database", "Photos share", "anna (#101) [Felicia]", "elsa (#100) [Felicia]"].sort()); assert.equal(restoreAlerts(g, tests, 0, NOW).filter((a) => !a.guest).length, 1, "with no limit only the failure"); }); }); describe("through the routes, the page, the alerts and the figures", () => { let app: Server; let base = ""; let who = "operator"; before(async () => { await db.insert(users).values([ { oidcSub: "admin", name: "Ada", role: "admin" }, { oidcSub: "operator", name: "Olga", role: "operator" }, { oidcSub: "viewer", name: "Vic", role: "viewer" }, ]); setCoverageLoaderForTests(async () => fixture()); const a = express(); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/backups", backupsRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/backups`; }); after(() => { setCoverageLoaderForTests(null); app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); const today = new Date().toISOString().slice(0, 10); test("an operator records a test; a viewer can't; a wrong one is refused", async () => { assert.equal((await send("POST", "/restore-tests", { key: "guest:1:100", label: "elsa", result: "ok", note: "booted and the data was there" })).status, 201); who = "viewer"; assert.equal((await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "ok" })).status, 403); who = "operator"; for (const bad of [ { key: "guest:1:101", label: "anna", result: "excellent" }, { key: "guest:one:two", label: "anna", result: "ok" }, { key: "../etc", label: "anna", result: "ok" }, { key: "guest:1:101", label: "anna", result: "ok", testedAt: "2999-01-01" }, { key: "guest:1:101", label: "anna", result: "ok", testedAt: "2026-02-30" }, { label: "!!!", result: "ok" }, { key: "guest:1:101", label: "", result: "ok" }, ]) assert.equal((await send("POST", "/restore-tests", bad)).status, 400, JSON.stringify(bad)); assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "restore_test" && e.action === "record").length, 1); }); test("the Backups page's data says how each guest's restore went, or that nobody has tried", async () => { const cov = await (await send("GET", "/coverage")).json(); const by = Object.fromEntries(cov.guests.map((g: { name: string }) => [g.name, g])); assert.equal(by.elsa.restoreKey, "guest:1:100"); assert.deepEqual([by.elsa.restoreTest.status, by.elsa.restoreTest.testedAt, by.elsa.restoreTest.note], ["ok", today, "booted and the data was there"]); assert.equal(by.anna.restoreTest.status, "never"); assert.equal(cov.restoreTestDays, 0); }); test("something that isn't a guest gets a key from its name, and is listed with how it stands", async () => { const res = await send("POST", "/restore-tests", { label: "Photos share", result: "ok", testedAt: daysAgo(40) }); assert.equal(res.status, 201); assert.equal((await res.json()).test.subjectKey, "custom:photos-share"); const list = await (await send("GET", "/restore-tests")).json(); assert.deepEqual(list.custom.map((c: { key: string; status: string }) => [c.key, c.status]), [["custom:photos-share", "ok"]]); assert.equal(list.tests.length, 2); }); test("a failed test is an alert straight away; with a limit, never-tested and old ones are too; marking 'not needed' ends it", async () => { await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "failed", note: "the disk wouldn't attach" }); invalidateAlertsCache(); let alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:")); assert.deepEqual(alerts.map((a) => [a.id, a.severity]), [["backup:restore-test:guest:1:101", "critical"]]); assert.match(alerts[0].message, /anna \(#101\).*failed — the disk wouldn't attach/); assert.equal(alerts[0].link, "/backups"); await updateSettings({ healthChecks: { restoreTestDays: 30 } }); invalidateAlertsCache(); alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:")); assert.deepEqual(alerts.map((a) => a.id).sort(), ["backup:restore-test:custom:photos-share", "backup:restore-test:guest:1:101"], "elsa is fine; the photos share was last tested 40 days ago"); await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "not_needed", note: "a scratch machine" }); invalidateAlertsCache(); alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:")); assert.deepEqual(alerts.map((a) => a.id), ["backup:restore-test:custom:photos-share"]); await updateSettings({ healthChecks: { restoreTestDays: 0 } }); }); test("metrics say how old each test is and whether it passed", async () => { const text = await buildMetrics(); assert.match(text, /homelab_restore_test_age_days\{subject="elsa"\} 0/); assert.match(text, /homelab_restore_test_age_days\{subject="Photos share"\} 40/); assert.match(text, /homelab_restore_test_passed\{subject="elsa"\} 1/); assert.doesNotMatch(text, /subject="anna"/, "'not needed' isn't measured"); }); test("only an administrator takes an entry out, and that is recorded", async () => { const [first] = await db.select().from(restoreTests); assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 403); who = "admin"; assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 204); assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 404); assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "restore_test" && e.action === "delete").length, 1); }); });