/** * Backup coverage: what the rows say for each kind of guest, and the route and the digest around them. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; import type { Server } from "node:http"; const dir = mkdtempSync(join(tmpdir(), "hm-coverage-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { buildCoverage, summarize, setCoverageLoaderForTests } = await import("../src/services/backupCoverage.js"); const { backupsRouter } = await import("../src/routes/backups.js"); const { buildDigest } = await import("../src/services/weeklyDigest.js"); await runMigrations(); const NOW = new Date("2026-10-17T12:00:00Z"); const HOUR = 3600_000; const guest = (vmid: number, name: string, type: "qemu" | "lxc" = "qemu", node = "pve1") => ({ vmid, name, node, type, status: "running", cpu: null, maxmem: null, mem: null, uptime: null }); const allJob = { id: "j1", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null }; const task = (guestId: string | null, ago: number, ok = true, status = ok ? "OK" : "job errors") => ({ node: "pve1", upid: `upid-${guestId}-${ago}`, guestId, status, ok, startTime: new Date(NOW.getTime() - ago * HOUR).toISOString(), endTime: new Date(NOW.getTime() - ago * HOUR + 600_000).toISOString(), }); const snap = (type: string, id: string, ago: number, verification: "ok" | "failed" | null = "ok") => ({ backupType: type, backupId: id, backupTime: Math.floor((NOW.getTime() - ago * HOUR) / 1000), verification }); const px = (guests: ReturnType[], jobs: (typeof allJob)[], tasks: ReturnType[]) => [{ integrationId: 1, integrationName: "Felicia", guests, jobs, tasks }]; describe("what each guest reads as", () => { test("backed up recently to a Backup Server: ok; a while ago: stale", () => { const rows = buildCoverage({ proxmox: px([guest(100, "web"), guest(101, "db", "lxc")], [allJob], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 5), snap("ct", "101", 100)] }], servers: [], now: NOW }); const by = Object.fromEntries(rows.map((r) => [r.name, r])); assert.equal(by.web.status, "ok"); assert.equal(by.web.lastSource, "pbs"); assert.equal(by.db.status, "stale", "100 hours is more than the 48 by default"); assert.equal(by.web.lastBackupAt, new Date(NOW.getTime() - 5 * HOUR).toISOString()); }); test("a task that names the guest counts, and the newer of the two sources wins", () => { const rows = buildCoverage({ proxmox: px([guest(100, "web")], [allJob], [task("100", 30), task("100", 3, false)]), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 10)] }], servers: [], now: NOW }); assert.equal(rows[0].lastSource, "pbs", "the snapshot (10 h) is newer than the successful task (30 h)"); assert.equal(rows[0].status, "failed", "but the latest run, 3 h ago, failed"); assert.match(rows[0].failure!, /job errors/); }); test("a failed run followed by a good backup is not a failure any more", () => { const rows = buildCoverage({ proxmox: px([guest(100, "web")], [allJob], [task("100", 8, false), task("100", 2, true)]), pbs: [], servers: [], now: NOW }); assert.equal(rows[0].status, "ok"); assert.equal(rows[0].lastSource, "task"); }); test("a snapshot that failed verification is a failure", () => { const rows = buildCoverage({ proxmox: px([guest(100, "web")], [allJob], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 2, "failed")] }], servers: [], now: NOW }); assert.equal(rows[0].status, "failed"); assert.match(rows[0].failure!, /verification/); }); test("in a job with nothing to say when it last ran is unknown; in no job and no backup is none", () => { const only101 = { ...allJob, allGuests: false, vmids: "100" }; const rows = buildCoverage({ proxmox: px([guest(100, "covered"), guest(101, "forgotten")], [only101], []), pbs: [], servers: [], now: NOW }); const by = Object.fromEntries(rows.map((r) => [r.name, r])); assert.equal(by.covered.status, "unknown"); assert.equal(by.covered.inJob, true); assert.equal(by.forgotten.status, "none"); assert.equal(by.forgotten.inJob, false); }); test("a guest in no job that still has an old backup is judged by its age", () => { const rows = buildCoverage({ proxmox: px([guest(101, "manual")], [], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "101", 6)] }], servers: [], now: NOW }); assert.equal(rows[0].status, "ok"); assert.equal(rows[0].inJob, false); }); test("how long counts as stale is up to the caller", () => { const input = { proxmox: px([guest(100, "web")], [allJob], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 100)] }], servers: [], now: NOW }; assert.equal(buildCoverage({ ...input, staleHours: 24 * 3 })[0].status, "stale"); assert.equal(buildCoverage({ ...input, staleHours: 24 * 7 })[0].status, "ok"); }); test("a VM and a container with the same number aren't mixed up, and a linked server is named", () => { const rows = buildCoverage({ proxmox: px([guest(100, "vm100", "qemu"), guest(100, "ct100", "lxc", "pve2")], [allJob], []), pbs: [{ integrationName: "PBS", groups: [snap("ct", "100", 1)] }], servers: [{ id: 7, name: "elsa", proxmoxIntegrationId: 1, proxmoxVmid: 100 }], now: NOW, }); const by = Object.fromEntries(rows.map((r) => [r.name, r])); assert.equal(by.ct100.status, "ok"); assert.equal(by.vm100.status, "unknown", "only the container has a snapshot"); assert.deepEqual(by.vm100.server, { id: 7, name: "elsa" }); }); test("the counts add up", () => { const rows = buildCoverage({ proxmox: px([guest(100, "a"), guest(101, "b")], [{ ...allJob, allGuests: false, vmids: "100" }], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 1)] }], servers: [], now: NOW }); assert.deepEqual(summarize(rows), { ok: 1, stale: 0, failed: 0, none: 1, unknown: 0, total: 2 }); }); }); describe("the route and the digest", () => { let app: Server; let base = ""; before(async () => { await db.insert(users).values({ oidcSub: "viewer", name: "Vic", role: "viewer" }); await db.insert(servers).values({ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", lastSeenAt: new Date().toISOString() }); setCoverageLoaderForTests(async () => ({ proxmox: px([guest(100, "web"), guest(101, "lonely")], [{ ...allJob, allGuests: false, vmids: "100" }], []).map((p) => ({ ...p })), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 100)] }], sources: [ { kind: "proxmox" as const, name: "Felicia", ok: true }, { kind: "pbs" as const, name: "PBS", ok: false, error: "couldn't read tank" }, ], })); const a = express(); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: "viewer" } }; next(); }); a.use("/api/backups", backupsRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/backups`; }); after(() => { setCoverageLoaderForTests(null); app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("anyone signed in gets the rows, the counts, and which sources couldn't be read", async () => { const body = await (await fetch(`${base}/coverage`)).json(); assert.equal(body.staleHours, 48); assert.deepEqual(body.guests.map((g: { name: string }) => g.name), ["lonely", "web"]); assert.equal(body.counts.total, 2); assert.equal(body.sources.find((s: { name: string }) => s.name === "PBS").ok, false); }); test("the stale threshold is set in days, and nonsense falls back to the default", async () => { // web's newest snapshot is 100 hours old: stale at 2 days, fine at 7. assert.equal((await (await fetch(`${base}/coverage?staleDays=7`)).json()).guests.find((g: { name: string }) => g.name === "web").status, "ok"); assert.equal((await (await fetch(`${base}/coverage?staleDays=2`)).json()).guests.find((g: { name: string }) => g.name === "web").status, "stale"); assert.equal((await (await fetch(`${base}/coverage?staleDays=banana`)).json()).staleHours, 48); assert.equal((await (await fetch(`${base}/coverage?staleDays=9999`)).json()).staleHours, 48); }); test("the weekly digest lists what isn't backed up, worst first", async () => { const digest = await buildDigest(new Date()); assert.match(digest.message, /BACKUPS\n0 of 2 guests have a recent backup\.\n- lonely \(VM 101\): no backup, and not in any backup job\n- web \(VM 100\): last backup \d+ d( \d+ h)? ago/); }); });