Files
Homelab-manager/server/test/restoreTests.test.ts
T
bobbanandClaude Sonnet 5.5 92726568b7 Endpoint checks, restore tests, and a change log
- Endpoints: HTTP, TCP and TLS checks with answer time, uptime and certificate expiry; failures and expiring certificates become alerts and notifications; linked to a server for maintenance and tags
- Restore tests: record when a backup was restored and how it went; failed tests always alert, overdue ones with a limit set; shown on the Backup coverage page
- Change log: one filterable timeline made from the audit log and the incident record

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-18 23:03:56 +02:00

211 lines
12 KiB
TypeScript

/**
* Restore tests: what the newest test of something amounts to, when to speak up, and the routes that record them.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-restore-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, restoreTests, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { restoreInfo, restoreAlerts, customKey, guestKey } = await import("../src/services/restoreTests.js");
const { setCoverageLoaderForTests, buildCoverage } = await import("../src/services/backupCoverage.js");
const { backupsRouter } = await import("../src/routes/backups.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { updateSettings } = await import("../src/services/settingsStore.js");
const { buildMetrics } = await import("../src/services/metrics.js");
await runMigrations();
const NOW = new Date("2026-10-18T12:00:00Z");
const daysAgo = (n: number) => new Date(NOW.getTime() - n * 86_400_000).toISOString().slice(0, 10);
const test1 = (result: string, ago: number, extra: Record<string, unknown> = {}) => ({ result, testedAt: daysAgo(ago), note: null as string | null, testedBy: "Ada", ...extra });
describe("what the newest test amounts to", () => {
test("how it went; none at all is never; an old pass is stale once there is a limit; a failure or 'not needed' doesn't age", () => {
assert.equal(restoreInfo(undefined, NOW, 90).status, "never");
assert.deepEqual(restoreInfo(test1("ok", 10), NOW, 90), { status: "ok", testedAt: daysAgo(10), ageDays: 10, note: null, testedBy: "Ada" });
assert.equal(restoreInfo(test1("ok", 100), NOW, 90).status, "stale");
assert.equal(restoreInfo(test1("ok", 100), NOW, 0).status, "ok", "no limit, no staleness");
assert.equal(restoreInfo(test1("partial", 100), NOW, 90).status, "stale");
assert.equal(restoreInfo(test1("failed", 400), NOW, 90).status, "failed");
assert.equal(restoreInfo(test1("not_needed", 400), NOW, 90).status, "not_needed");
});
test("something that isn't a guest is told apart by its name", () => {
assert.equal(customKey("Photos share"), "custom:photos-share");
assert.equal(customKey(" Gitea — database! "), "custom:gitea-database");
assert.equal(customKey("Åkerblom's NAS"), "custom:akerblom-s-nas");
assert.equal(customKey("!!!"), null);
assert.equal(guestKey(2, 105), "guest:2:105");
});
});
const guest = (vmid: number, name: string) => ({ vmid, name, node: "pve1", type: "qemu" as const, status: "running", cpu: null, maxmem: null, mem: null, uptime: null });
const fixture = () => {
const loaded = {
proxmox: [{ integrationId: 1, integrationName: "Felicia", guests: [guest(100, "elsa"), guest(101, "anna"), guest(102, "zoe")], jobs: [{ id: "j", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null }], tasks: [] }],
pbs: [{ integrationName: "Backup server", groups: [{ backupType: "vm", backupId: "100", backupTime: Math.floor(Date.now() / 1000) - 3600, verification: "ok" as const }, { backupType: "vm", backupId: "101", backupTime: Math.floor(Date.now() / 1000) - 7200, verification: "ok" as const }] }],
sources: [{ kind: "proxmox" as const, name: "Felicia", ok: true }],
};
return loaded;
};
describe("when to speak up", () => {
const report = () => ({ guests: buildCoverage({ ...fixture(), servers: [], now: NOW }), counts: {} as never, sources: [], staleHours: 48, generatedAt: NOW.toISOString() });
const row = (key: string, label: string, result: string, ago: number) => [key, { id: 1, subjectKey: key, label, testedAt: daysAgo(ago), result, note: null, testedBy: null, createdAt: "" }] as const;
test("with no limit only a failure speaks; with one, a guest that has backups and no recent test does too", () => {
const g = report();
assert.deepEqual(restoreAlerts(g, new Map(), 0, NOW), [], "no limit, nothing tested: quiet");
const limited = restoreAlerts(g, new Map(), 90, NOW);
assert.deepEqual(limited.map((a) => [a.guest?.name, a.severity]), [["elsa", "warning"], ["anna", "warning"]], "zoe has no backups to restore from, so nothing to say");
assert.match(limited[0].message, /elsa \(#100\) \[Felicia\]: a restore has never been tested/);
const failed = restoreAlerts(g, new Map([row("guest:1:100", "elsa", "failed", 3)]), 0, NOW);
assert.equal(failed.length, 1);
assert.equal(failed[0].severity, "critical");
assert.match(failed[0].message, /the last restore test \(.*\) failed/);
});
test("a recent pass, 'not needed' and an old pass under the limit are all fine; an old one isn't", () => {
const g = report();
const tests = new Map([row("guest:1:100", "elsa", "ok", 10), row("guest:1:101", "anna", "not_needed", 500)]);
assert.deepEqual(restoreAlerts(g, tests, 90, NOW), []);
const old = restoreAlerts(g, new Map([row("guest:1:100", "elsa", "ok", 120), row("guest:1:101", "anna", "ok", 1)]), 90, NOW);
assert.equal(old.length, 1);
assert.match(old[0].message, /last restore test was 120 days ago \(the limit is 90 days\)/);
});
test("something that isn't a guest is told of when it failed, or when it is stale", () => {
const g = report();
const tests = new Map([row("custom:photos", "Photos share", "ok", 200), row("custom:db", "Gitea database", "failed", 5), row("custom:fine", "Fine", "ok", 3), row("guest:9:9", "gone guest", "ok", 999)]);
const out = restoreAlerts(g, tests, 90, NOW);
assert.deepEqual(out.map((a) => a.message.split(":")[0]).sort(), ["Gitea database", "Photos share", "anna (#101) [Felicia]", "elsa (#100) [Felicia]"].sort());
assert.equal(restoreAlerts(g, tests, 0, NOW).filter((a) => !a.guest).length, 1, "with no limit only the failure");
});
});
describe("through the routes, the page, the alerts and the figures", () => {
let app: Server;
let base = "";
let who = "operator";
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
setCoverageLoaderForTests(async () => fixture());
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/backups", backupsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/backups`;
});
after(() => {
setCoverageLoaderForTests(null);
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
const today = new Date().toISOString().slice(0, 10);
test("an operator records a test; a viewer can't; a wrong one is refused", async () => {
assert.equal((await send("POST", "/restore-tests", { key: "guest:1:100", label: "elsa", result: "ok", note: "booted and the data was there" })).status, 201);
who = "viewer";
assert.equal((await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "ok" })).status, 403);
who = "operator";
for (const bad of [
{ key: "guest:1:101", label: "anna", result: "excellent" },
{ key: "guest:one:two", label: "anna", result: "ok" },
{ key: "../etc", label: "anna", result: "ok" },
{ key: "guest:1:101", label: "anna", result: "ok", testedAt: "2999-01-01" },
{ key: "guest:1:101", label: "anna", result: "ok", testedAt: "2026-02-30" },
{ label: "!!!", result: "ok" },
{ key: "guest:1:101", label: "", result: "ok" },
]) assert.equal((await send("POST", "/restore-tests", bad)).status, 400, JSON.stringify(bad));
assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "restore_test" && e.action === "record").length, 1);
});
test("the Backups page's data says how each guest's restore went, or that nobody has tried", async () => {
const cov = await (await send("GET", "/coverage")).json();
const by = Object.fromEntries(cov.guests.map((g: { name: string }) => [g.name, g]));
assert.equal(by.elsa.restoreKey, "guest:1:100");
assert.deepEqual([by.elsa.restoreTest.status, by.elsa.restoreTest.testedAt, by.elsa.restoreTest.note], ["ok", today, "booted and the data was there"]);
assert.equal(by.anna.restoreTest.status, "never");
assert.equal(cov.restoreTestDays, 0);
});
test("something that isn't a guest gets a key from its name, and is listed with how it stands", async () => {
const res = await send("POST", "/restore-tests", { label: "Photos share", result: "ok", testedAt: daysAgo(40) });
assert.equal(res.status, 201);
assert.equal((await res.json()).test.subjectKey, "custom:photos-share");
const list = await (await send("GET", "/restore-tests")).json();
assert.deepEqual(list.custom.map((c: { key: string; status: string }) => [c.key, c.status]), [["custom:photos-share", "ok"]]);
assert.equal(list.tests.length, 2);
});
test("a failed test is an alert straight away; with a limit, never-tested and old ones are too; marking 'not needed' ends it", async () => {
await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "failed", note: "the disk wouldn't attach" });
invalidateAlertsCache();
let alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:"));
assert.deepEqual(alerts.map((a) => [a.id, a.severity]), [["backup:restore-test:guest:1:101", "critical"]]);
assert.match(alerts[0].message, /anna \(#101\).*failed — the disk wouldn't attach/);
assert.equal(alerts[0].link, "/backups");
await updateSettings({ healthChecks: { restoreTestDays: 30 } });
invalidateAlertsCache();
alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:"));
assert.deepEqual(alerts.map((a) => a.id).sort(), ["backup:restore-test:custom:photos-share", "backup:restore-test:guest:1:101"], "elsa is fine; the photos share was last tested 40 days ago");
await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "not_needed", note: "a scratch machine" });
invalidateAlertsCache();
alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:"));
assert.deepEqual(alerts.map((a) => a.id), ["backup:restore-test:custom:photos-share"]);
await updateSettings({ healthChecks: { restoreTestDays: 0 } });
});
test("metrics say how old each test is and whether it passed", async () => {
const text = await buildMetrics();
assert.match(text, /homelab_restore_test_age_days\{subject="elsa"\} 0/);
assert.match(text, /homelab_restore_test_age_days\{subject="Photos share"\} 40/);
assert.match(text, /homelab_restore_test_passed\{subject="elsa"\} 1/);
assert.doesNotMatch(text, /subject="anna"/, "'not needed' isn't measured");
});
test("only an administrator takes an entry out, and that is recorded", async () => {
const [first] = await db.select().from(restoreTests);
assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 403);
who = "admin";
assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 204);
assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 404);
assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "restore_test" && e.action === "delete").length, 1);
});
});