UniFi: the controller's client list is matched to servers by IP address, so a server is connected through a switch or access point (and port). That is a dependency made by itself, shown on the server page, the Network devices page and the Connections map. IP Addresses can sync the devices' addresses. A server's page is split into Overview, Health, Network, History, Tasks and Settings, by ?view=; alerts link to the section they are about. My work lists what has your name on it: alerts you have, maintenance you started (extend, end with undo) and update runs you started, with a badge. Settings has a search box (and the command palette lists every setting). Removing a dependency, admin link, runbook, hand-made IP address, port opening or restore test is undone from the message instead of asked about first. Two backup tests used a fixed date and failed as days passed; they follow the real clock now. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
186 lines
11 KiB
TypeScript
186 lines
11 KiB
TypeScript
/**
|
|
* Guests whose newest backup has gone too old: the limit (Settings, or a server's own), the alert, what silences it, the daily list and /metrics.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-stale-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const { db } = await import("../src/db/client.js");
|
|
const { servers, maintenanceWindows, integrations } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const { buildCoverage, staleBackups, collectStaleBackups, getBackupCoverage, setCoverageLoaderForTests } = await import("../src/services/backupCoverage.js");
|
|
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
|
const { buildMetrics } = await import("../src/services/metrics.js");
|
|
const { updateSettings } = await import("../src/services/settingsStore.js");
|
|
const { notifyStaleBackups } = await import("../src/services/notify.js");
|
|
const { hashToken } = await import("../src/services/tokens.js");
|
|
|
|
await runMigrations();
|
|
after(() => {
|
|
setCoverageLoaderForTests(null);
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows keeps the database file locked until the process exits.
|
|
}
|
|
});
|
|
|
|
const HOUR = 3600_000;
|
|
const guest = (vmid: number, name: string) => ({ vmid, name, node: "pve1", type: "qemu" as const, status: "running", cpu: null, maxmem: null, mem: null, uptime: null });
|
|
const allJob = { id: "j1", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null };
|
|
const px = (guests: ReturnType<typeof guest>[]) => [{ integrationId: 1, integrationName: "Felicia", guests, jobs: [allJob], tasks: [] }];
|
|
const snapAgo = (now: number, vmid: number, hours: number) => ({ backupType: "vm", backupId: String(vmid), backupTime: Math.floor((now - hours * HOUR) / 1000), verification: "ok" as const });
|
|
|
|
describe("when a guest is stale", () => {
|
|
const NOW = new Date(); // the real clock: the routes under test read it too
|
|
const now = NOW.getTime();
|
|
const input = (hoursAgo: number, extra: { servers?: Parameters<typeof buildCoverage>[0]["servers"]; staleHours?: number } = {}) => ({
|
|
proxmox: px([guest(100, "web")]),
|
|
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, hoursAgo)] }],
|
|
servers: extra.servers ?? [],
|
|
now: NOW,
|
|
staleHours: extra.staleHours,
|
|
});
|
|
|
|
test("a guest older than the limit is stale, and the limit it was judged by is on the row", () => {
|
|
const [g] = buildCoverage(input(60));
|
|
assert.equal(g.status, "stale");
|
|
assert.equal(g.staleAfterHours, 48);
|
|
assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].status, "ok");
|
|
assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].staleAfterHours, 72);
|
|
});
|
|
|
|
test("a server that is this guest can have its own limit — longer for a weekly backup, or 0 for never", () => {
|
|
const linked = (staleHours: number | null) => [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100, staleHours }];
|
|
assert.equal(buildCoverage(input(200, { servers: linked(240) }))[0].status, "ok", "eight days is fine for a limit of ten");
|
|
assert.equal(buildCoverage(input(300, { servers: linked(240) }))[0].status, "stale");
|
|
assert.equal(buildCoverage(input(5000, { servers: linked(0) }))[0].status, "ok", "0 means this one is never called stale");
|
|
assert.equal(buildCoverage(input(200, { servers: linked(null) }))[0].status, "stale", "no limit of its own: the global one");
|
|
// a failure is a failure whatever the limit says
|
|
const failed = buildCoverage({ ...input(1, { servers: linked(0) }), pbs: [{ integrationName: "PBS", groups: [{ ...snapAgo(now, 100, 1), verification: "failed" as const }] }] });
|
|
assert.equal(failed[0].status, "failed");
|
|
});
|
|
|
|
test("only stale guests are listed, oldest first, critical at three times the limit", () => {
|
|
const report = {
|
|
guests: buildCoverage({
|
|
proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "fresh"), guest(103, "never")]),
|
|
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 160), snapAgo(now, 102, 2)] }],
|
|
servers: [],
|
|
now: NOW,
|
|
}),
|
|
counts: { ok: 1, stale: 2, failed: 0, none: 0, unknown: 1, total: 4 },
|
|
sources: [],
|
|
staleHours: 48,
|
|
generatedAt: NOW.toISOString(),
|
|
};
|
|
const stale = staleBackups(report, NOW);
|
|
assert.deepEqual(stale.map((s) => [s.guest.name, s.severity]), [["db", "critical"], ["web", "warning"]]);
|
|
assert.equal(stale[0].key, "backup-stale:1:101");
|
|
assert.match(stale[0].message, /db \(#101\) \[Felicia\]: its newest backup is 6 days old \(the limit is 48 h\)/);
|
|
assert.match(stale[1].message, /60 h old \(the limit is 48 h\)/);
|
|
});
|
|
|
|
test("it is silenced by maintenance on the Proxmox integration or on the server the guest is", () => {
|
|
const base = { guests: buildCoverage({ ...input(60), servers: [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100 }] }), counts: { ok: 0, stale: 1, failed: 0, none: 0, unknown: 0, total: 1 }, sources: [], staleHours: 48, generatedAt: NOW.toISOString() };
|
|
assert.equal(staleBackups(base, NOW)[0].silenced, false);
|
|
assert.equal(staleBackups(base, NOW, new Set(["integration:1"]))[0].silenced, true);
|
|
assert.equal(staleBackups(base, NOW, new Set(["server:7"]))[0].silenced, true);
|
|
assert.equal(staleBackups(base, NOW, new Set(["server:8", "integration:2"]))[0].silenced, false);
|
|
});
|
|
});
|
|
|
|
describe("through the alerts, the daily list and /metrics", () => {
|
|
before(async () => {
|
|
// The guests are Proxmox's, read from a stand-in; the integration they belong to still has to exist for a server to be linked to one.
|
|
await db.insert(integrations).values({ id: 1, type: "proxmox", name: "Felicia", baseUrl: "https://pve.example", enabled: false });
|
|
});
|
|
|
|
// The alert pass judges by the real clock, so these backups are dated from it.
|
|
const fixture = () => {
|
|
const now = Date.now();
|
|
return {
|
|
proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "weekly"), guest(103, "fine")]).map((p) => ({ ...p })),
|
|
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 200), snapAgo(now, 102, 150), snapAgo(now, 103, 3)] }],
|
|
sources: [{ kind: "proxmox" as const, name: "Felicia", ok: true }, { kind: "pbs" as const, name: "PBS", ok: true }],
|
|
};
|
|
};
|
|
const staleAlerts = async () => {
|
|
invalidateAlertsCache();
|
|
return (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:backup-stale:"));
|
|
};
|
|
|
|
test("each stale guest is an alert of the backup kind, with a link, a target and a severity", async () => {
|
|
setCoverageLoaderForTests(async () => fixture());
|
|
const alerts = await staleAlerts();
|
|
assert.deepEqual(alerts.map((a) => a.id).sort(), ["backup:backup-stale:1:100", "backup:backup-stale:1:101", "backup:backup-stale:1:102"]);
|
|
const db101 = alerts.find((a) => a.id.endsWith(":101"))!;
|
|
assert.equal(db101.category, "backup");
|
|
assert.equal(db101.severity, "critical", "200 hours is more than three times 48");
|
|
assert.equal(alerts.find((a) => a.id.endsWith(":100"))!.severity, "warning");
|
|
assert.equal(db101.link, "/backups");
|
|
assert.deepEqual(db101.target, { type: "integration", id: 1, name: "Felicia" });
|
|
assert.equal(db101.silenced, false);
|
|
});
|
|
|
|
test("a server's own limit changes its guest's alert, and that server's maintenance silences it", async () => {
|
|
const [s] = await db.insert(servers).values({ name: "weekly", apiTokenHash: hashToken("t"), apiTokenPrefix: "t", proxmoxIntegrationId: 1, proxmoxVmid: 102, thresholds: JSON.stringify({ backupStaleHours: 240 }) }).returning();
|
|
setCoverageLoaderForTests(async () => fixture());
|
|
let alerts = await staleAlerts();
|
|
assert.equal(alerts.some((a) => a.id.endsWith(":102")), false, "150 hours is fine for a limit of 240");
|
|
await db.update(servers).set({ thresholds: JSON.stringify({ backupStaleHours: 100 }) });
|
|
alerts = await staleAlerts();
|
|
const weekly = alerts.find((a) => a.id.endsWith(":102"))!;
|
|
assert.ok(weekly, "150 hours is over a limit of 100");
|
|
assert.deepEqual(weekly.target && { type: weekly.target.type, id: weekly.target.id }, { type: "server", id: s.id }, "it is the server that can be silenced");
|
|
|
|
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: s.id, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
|
|
assert.equal((await staleAlerts()).find((a) => a.id.endsWith(":102"))!.silenced, true);
|
|
await db.delete(maintenanceWindows);
|
|
await db.update(servers).set({ thresholds: null });
|
|
});
|
|
|
|
test("the limit in Settings is what the others follow, and 0 turns the alerts off", async () => {
|
|
setCoverageLoaderForTests(async () => fixture());
|
|
await updateSettings({ healthChecks: { backupStaleHours: 100 } });
|
|
assert.deepEqual((await staleAlerts()).map((a) => a.id.split(":").pop()).sort(), ["101", "102"], "60 hours is fine for a limit of 100");
|
|
await updateSettings({ healthChecks: { backupStaleHours: 0 } });
|
|
assert.deepEqual(await staleAlerts(), []);
|
|
await updateSettings({ healthChecks: { backupStaleHours: 48 } });
|
|
});
|
|
|
|
test("the daily list is the stale guests that aren't under maintenance", async () => {
|
|
setCoverageLoaderForTests(async () => fixture());
|
|
const list = await collectStaleBackups();
|
|
assert.equal(list.length, 3);
|
|
await db.insert(maintenanceWindows).values({ targetType: "integration", targetId: 1, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
|
|
assert.deepEqual(await collectStaleBackups(), [], "the whole Proxmox integration is being worked on");
|
|
await db.delete(maintenanceWindows);
|
|
await notifyStaleBackups([]); // nothing to say, nothing sent, and no error with no channel set up
|
|
await notifyStaleBackups(list);
|
|
});
|
|
|
|
test("/metrics says how many guests are in each state and how old each newest backup is, from what was last read", async () => {
|
|
setCoverageLoaderForTests(async () => fixture());
|
|
invalidateAlertsCache();
|
|
const text = await buildMetrics();
|
|
assert.match(text, /homelab_backup_guests\{status="ok"\} 1/);
|
|
assert.match(text, /homelab_backup_guests\{status="stale"\} 3/);
|
|
const age = /homelab_backup_last_age_seconds\{guest="web",vmid="100",integration="Felicia"\} (\d+)/.exec(text);
|
|
assert.ok(age, "an age for each guest with a backup");
|
|
assert.ok(Math.abs(Number(age[1]) - 60 * 3600) < 120, `about 60 hours, got ${age[1]}`);
|
|
// the page's own selector still asks for its own limit for every guest
|
|
assert.equal((await getBackupCoverage({ staleHours: 24 * 14 })).counts.stale, 0);
|
|
});
|
|
});
|