/** * Guests whose newest backup has gone too old: the limit (Settings, or a server's own), the alert, what silences it, the daily list and /metrics. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; const dir = mkdtempSync(join(tmpdir(), "hm-stale-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const { db } = await import("../src/db/client.js"); const { servers, maintenanceWindows, integrations } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { buildCoverage, staleBackups, collectStaleBackups, getBackupCoverage, setCoverageLoaderForTests } = await import("../src/services/backupCoverage.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { buildMetrics } = await import("../src/services/metrics.js"); const { updateSettings } = await import("../src/services/settingsStore.js"); const { notifyStaleBackups } = await import("../src/services/notify.js"); const { hashToken } = await import("../src/services/tokens.js"); await runMigrations(); after(() => { setCoverageLoaderForTests(null); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); const HOUR = 3600_000; const guest = (vmid: number, name: string) => ({ vmid, name, node: "pve1", type: "qemu" as const, status: "running", cpu: null, maxmem: null, mem: null, uptime: null }); const allJob = { id: "j1", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null }; const px = (guests: ReturnType[]) => [{ integrationId: 1, integrationName: "Felicia", guests, jobs: [allJob], tasks: [] }]; const snapAgo = (now: number, vmid: number, hours: number) => ({ backupType: "vm", backupId: String(vmid), backupTime: Math.floor((now - hours * HOUR) / 1000), verification: "ok" as const }); describe("when a guest is stale", () => { const NOW = new Date("2026-10-17T12:00:00Z"); const now = NOW.getTime(); const input = (hoursAgo: number, extra: { servers?: Parameters[0]["servers"]; staleHours?: number } = {}) => ({ proxmox: px([guest(100, "web")]), pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, hoursAgo)] }], servers: extra.servers ?? [], now: NOW, staleHours: extra.staleHours, }); test("a guest older than the limit is stale, and the limit it was judged by is on the row", () => { const [g] = buildCoverage(input(60)); assert.equal(g.status, "stale"); assert.equal(g.staleAfterHours, 48); assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].status, "ok"); assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].staleAfterHours, 72); }); test("a server that is this guest can have its own limit — longer for a weekly backup, or 0 for never", () => { const linked = (staleHours: number | null) => [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100, staleHours }]; assert.equal(buildCoverage(input(200, { servers: linked(240) }))[0].status, "ok", "eight days is fine for a limit of ten"); assert.equal(buildCoverage(input(300, { servers: linked(240) }))[0].status, "stale"); assert.equal(buildCoverage(input(5000, { servers: linked(0) }))[0].status, "ok", "0 means this one is never called stale"); assert.equal(buildCoverage(input(200, { servers: linked(null) }))[0].status, "stale", "no limit of its own: the global one"); // a failure is a failure whatever the limit says const failed = buildCoverage({ ...input(1, { servers: linked(0) }), pbs: [{ integrationName: "PBS", groups: [{ ...snapAgo(now, 100, 1), verification: "failed" as const }] }] }); assert.equal(failed[0].status, "failed"); }); test("only stale guests are listed, oldest first, critical at three times the limit", () => { const report = { guests: buildCoverage({ proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "fresh"), guest(103, "never")]), pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 160), snapAgo(now, 102, 2)] }], servers: [], now: NOW, }), counts: { ok: 1, stale: 2, failed: 0, none: 0, unknown: 1, total: 4 }, sources: [], staleHours: 48, generatedAt: NOW.toISOString(), }; const stale = staleBackups(report, NOW); assert.deepEqual(stale.map((s) => [s.guest.name, s.severity]), [["db", "critical"], ["web", "warning"]]); assert.equal(stale[0].key, "backup-stale:1:101"); assert.match(stale[0].message, /db \(#101\) \[Felicia\]: its newest backup is 6 days old \(the limit is 48 h\)/); assert.match(stale[1].message, /60 h old \(the limit is 48 h\)/); }); test("it is silenced by maintenance on the Proxmox integration or on the server the guest is", () => { const base = { guests: buildCoverage({ ...input(60), servers: [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100 }] }), counts: { ok: 0, stale: 1, failed: 0, none: 0, unknown: 0, total: 1 }, sources: [], staleHours: 48, generatedAt: NOW.toISOString() }; assert.equal(staleBackups(base, NOW)[0].silenced, false); assert.equal(staleBackups(base, NOW, new Set(["integration:1"]))[0].silenced, true); assert.equal(staleBackups(base, NOW, new Set(["server:7"]))[0].silenced, true); assert.equal(staleBackups(base, NOW, new Set(["server:8", "integration:2"]))[0].silenced, false); }); }); describe("through the alerts, the daily list and /metrics", () => { before(async () => { // The guests are Proxmox's, read from a stand-in; the integration they belong to still has to exist for a server to be linked to one. await db.insert(integrations).values({ id: 1, type: "proxmox", name: "Felicia", baseUrl: "https://pve.example", enabled: false }); }); // The alert pass judges by the real clock, so these backups are dated from it. const fixture = () => { const now = Date.now(); return { proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "weekly"), guest(103, "fine")]).map((p) => ({ ...p })), pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 200), snapAgo(now, 102, 150), snapAgo(now, 103, 3)] }], sources: [{ kind: "proxmox" as const, name: "Felicia", ok: true }, { kind: "pbs" as const, name: "PBS", ok: true }], }; }; const staleAlerts = async () => { invalidateAlertsCache(); return (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:backup-stale:")); }; test("each stale guest is an alert of the backup kind, with a link, a target and a severity", async () => { setCoverageLoaderForTests(async () => fixture()); const alerts = await staleAlerts(); assert.deepEqual(alerts.map((a) => a.id).sort(), ["backup:backup-stale:1:100", "backup:backup-stale:1:101", "backup:backup-stale:1:102"]); const db101 = alerts.find((a) => a.id.endsWith(":101"))!; assert.equal(db101.category, "backup"); assert.equal(db101.severity, "critical", "200 hours is more than three times 48"); assert.equal(alerts.find((a) => a.id.endsWith(":100"))!.severity, "warning"); assert.equal(db101.link, "/backups"); assert.deepEqual(db101.target, { type: "integration", id: 1, name: "Felicia" }); assert.equal(db101.silenced, false); }); test("a server's own limit changes its guest's alert, and that server's maintenance silences it", async () => { const [s] = await db.insert(servers).values({ name: "weekly", apiTokenHash: hashToken("t"), apiTokenPrefix: "t", proxmoxIntegrationId: 1, proxmoxVmid: 102, thresholds: JSON.stringify({ backupStaleHours: 240 }) }).returning(); setCoverageLoaderForTests(async () => fixture()); let alerts = await staleAlerts(); assert.equal(alerts.some((a) => a.id.endsWith(":102")), false, "150 hours is fine for a limit of 240"); await db.update(servers).set({ thresholds: JSON.stringify({ backupStaleHours: 100 }) }); alerts = await staleAlerts(); const weekly = alerts.find((a) => a.id.endsWith(":102"))!; assert.ok(weekly, "150 hours is over a limit of 100"); assert.deepEqual(weekly.target && { type: weekly.target.type, id: weekly.target.id }, { type: "server", id: s.id }, "it is the server that can be silenced"); await db.insert(maintenanceWindows).values({ targetType: "server", targetId: s.id, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() }); assert.equal((await staleAlerts()).find((a) => a.id.endsWith(":102"))!.silenced, true); await db.delete(maintenanceWindows); await db.update(servers).set({ thresholds: null }); }); test("the limit in Settings is what the others follow, and 0 turns the alerts off", async () => { setCoverageLoaderForTests(async () => fixture()); await updateSettings({ healthChecks: { backupStaleHours: 100 } }); assert.deepEqual((await staleAlerts()).map((a) => a.id.split(":").pop()).sort(), ["101", "102"], "60 hours is fine for a limit of 100"); await updateSettings({ healthChecks: { backupStaleHours: 0 } }); assert.deepEqual(await staleAlerts(), []); await updateSettings({ healthChecks: { backupStaleHours: 48 } }); }); test("the daily list is the stale guests that aren't under maintenance", async () => { setCoverageLoaderForTests(async () => fixture()); const list = await collectStaleBackups(); assert.equal(list.length, 3); await db.insert(maintenanceWindows).values({ targetType: "integration", targetId: 1, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() }); assert.deepEqual(await collectStaleBackups(), [], "the whole Proxmox integration is being worked on"); await db.delete(maintenanceWindows); await notifyStaleBackups([]); // nothing to say, nothing sent, and no error with no channel set up await notifyStaleBackups(list); }); test("/metrics says how many guests are in each state and how old each newest backup is, from what was last read", async () => { setCoverageLoaderForTests(async () => fixture()); invalidateAlertsCache(); const text = await buildMetrics(); assert.match(text, /homelab_backup_guests\{status="ok"\} 1/); assert.match(text, /homelab_backup_guests\{status="stale"\} 3/); const age = /homelab_backup_last_age_seconds\{guest="web",vmid="100",integration="Felicia"\} (\d+)/.exec(text); assert.ok(age, "an age for each guest with a backup"); assert.ok(Math.abs(Number(age[1]) - 60 * 3600) < 120, `about 60 hours, got ${age[1]}`); // the page's own selector still asks for its own limit for every guest assert.equal((await getBackupCoverage({ staleHours: 24 * 14 })).counts.stale, 0); }); });