Stale-backup alerts, per-server alert limits, and update runs from the Updates page
- Backups: a guest whose newest backup is older than its limit is an alert (backup kind), in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds) - Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent, forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the global ones were; saving clears the alerts cache - Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and reports what is left and whether a restart is due; poller survives restarts - Notification toggle for update-run results; tests and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5dc6cea69c
commit
7a5b623444
42 files changed
+7561
-57
No files matched your search
@@ -0,0 +1,185 @@
|
||||
/**
|
||||
* Guests whose newest backup has gone too old: the limit (Settings, or a server's own), the alert, what silences it, the daily list and /metrics.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-stale-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { servers, maintenanceWindows, integrations } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { buildCoverage, staleBackups, collectStaleBackups, getBackupCoverage, setCoverageLoaderForTests } = await import("../src/services/backupCoverage.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { notifyStaleBackups } = await import("../src/services/notify.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
|
||||
await runMigrations();
|
||||
after(() => {
|
||||
setCoverageLoaderForTests(null);
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const HOUR = 3600_000;
|
||||
const guest = (vmid: number, name: string) => ({ vmid, name, node: "pve1", type: "qemu" as const, status: "running", cpu: null, maxmem: null, mem: null, uptime: null });
|
||||
const allJob = { id: "j1", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null };
|
||||
const px = (guests: ReturnType<typeof guest>[]) => [{ integrationId: 1, integrationName: "Felicia", guests, jobs: [allJob], tasks: [] }];
|
||||
const snapAgo = (now: number, vmid: number, hours: number) => ({ backupType: "vm", backupId: String(vmid), backupTime: Math.floor((now - hours * HOUR) / 1000), verification: "ok" as const });
|
||||
|
||||
describe("when a guest is stale", () => {
|
||||
const NOW = new Date("2026-10-17T12:00:00Z");
|
||||
const now = NOW.getTime();
|
||||
const input = (hoursAgo: number, extra: { servers?: Parameters<typeof buildCoverage>[0]["servers"]; staleHours?: number } = {}) => ({
|
||||
proxmox: px([guest(100, "web")]),
|
||||
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, hoursAgo)] }],
|
||||
servers: extra.servers ?? [],
|
||||
now: NOW,
|
||||
staleHours: extra.staleHours,
|
||||
});
|
||||
|
||||
test("a guest older than the limit is stale, and the limit it was judged by is on the row", () => {
|
||||
const [g] = buildCoverage(input(60));
|
||||
assert.equal(g.status, "stale");
|
||||
assert.equal(g.staleAfterHours, 48);
|
||||
assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].status, "ok");
|
||||
assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].staleAfterHours, 72);
|
||||
});
|
||||
|
||||
test("a server that is this guest can have its own limit — longer for a weekly backup, or 0 for never", () => {
|
||||
const linked = (staleHours: number | null) => [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100, staleHours }];
|
||||
assert.equal(buildCoverage(input(200, { servers: linked(240) }))[0].status, "ok", "eight days is fine for a limit of ten");
|
||||
assert.equal(buildCoverage(input(300, { servers: linked(240) }))[0].status, "stale");
|
||||
assert.equal(buildCoverage(input(5000, { servers: linked(0) }))[0].status, "ok", "0 means this one is never called stale");
|
||||
assert.equal(buildCoverage(input(200, { servers: linked(null) }))[0].status, "stale", "no limit of its own: the global one");
|
||||
// a failure is a failure whatever the limit says
|
||||
const failed = buildCoverage({ ...input(1, { servers: linked(0) }), pbs: [{ integrationName: "PBS", groups: [{ ...snapAgo(now, 100, 1), verification: "failed" as const }] }] });
|
||||
assert.equal(failed[0].status, "failed");
|
||||
});
|
||||
|
||||
test("only stale guests are listed, oldest first, critical at three times the limit", () => {
|
||||
const report = {
|
||||
guests: buildCoverage({
|
||||
proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "fresh"), guest(103, "never")]),
|
||||
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 160), snapAgo(now, 102, 2)] }],
|
||||
servers: [],
|
||||
now: NOW,
|
||||
}),
|
||||
counts: { ok: 1, stale: 2, failed: 0, none: 0, unknown: 1, total: 4 },
|
||||
sources: [],
|
||||
staleHours: 48,
|
||||
generatedAt: NOW.toISOString(),
|
||||
};
|
||||
const stale = staleBackups(report, NOW);
|
||||
assert.deepEqual(stale.map((s) => [s.guest.name, s.severity]), [["db", "critical"], ["web", "warning"]]);
|
||||
assert.equal(stale[0].key, "backup-stale:1:101");
|
||||
assert.match(stale[0].message, /db \(#101\) \[Felicia\]: its newest backup is 6 days old \(the limit is 48 h\)/);
|
||||
assert.match(stale[1].message, /60 h old \(the limit is 48 h\)/);
|
||||
});
|
||||
|
||||
test("it is silenced by maintenance on the Proxmox integration or on the server the guest is", () => {
|
||||
const base = { guests: buildCoverage({ ...input(60), servers: [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100 }] }), counts: { ok: 0, stale: 1, failed: 0, none: 0, unknown: 0, total: 1 }, sources: [], staleHours: 48, generatedAt: NOW.toISOString() };
|
||||
assert.equal(staleBackups(base, NOW)[0].silenced, false);
|
||||
assert.equal(staleBackups(base, NOW, new Set(["integration:1"]))[0].silenced, true);
|
||||
assert.equal(staleBackups(base, NOW, new Set(["server:7"]))[0].silenced, true);
|
||||
assert.equal(staleBackups(base, NOW, new Set(["server:8", "integration:2"]))[0].silenced, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the alerts, the daily list and /metrics", () => {
|
||||
before(async () => {
|
||||
// The guests are Proxmox's, read from a stand-in; the integration they belong to still has to exist for a server to be linked to one.
|
||||
await db.insert(integrations).values({ id: 1, type: "proxmox", name: "Felicia", baseUrl: "https://pve.example", enabled: false });
|
||||
});
|
||||
|
||||
// The alert pass judges by the real clock, so these backups are dated from it.
|
||||
const fixture = () => {
|
||||
const now = Date.now();
|
||||
return {
|
||||
proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "weekly"), guest(103, "fine")]).map((p) => ({ ...p })),
|
||||
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 200), snapAgo(now, 102, 150), snapAgo(now, 103, 3)] }],
|
||||
sources: [{ kind: "proxmox" as const, name: "Felicia", ok: true }, { kind: "pbs" as const, name: "PBS", ok: true }],
|
||||
};
|
||||
};
|
||||
const staleAlerts = async () => {
|
||||
invalidateAlertsCache();
|
||||
return (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:backup-stale:"));
|
||||
};
|
||||
|
||||
test("each stale guest is an alert of the backup kind, with a link, a target and a severity", async () => {
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
const alerts = await staleAlerts();
|
||||
assert.deepEqual(alerts.map((a) => a.id).sort(), ["backup:backup-stale:1:100", "backup:backup-stale:1:101", "backup:backup-stale:1:102"]);
|
||||
const db101 = alerts.find((a) => a.id.endsWith(":101"))!;
|
||||
assert.equal(db101.category, "backup");
|
||||
assert.equal(db101.severity, "critical", "200 hours is more than three times 48");
|
||||
assert.equal(alerts.find((a) => a.id.endsWith(":100"))!.severity, "warning");
|
||||
assert.equal(db101.link, "/backups");
|
||||
assert.deepEqual(db101.target, { type: "integration", id: 1, name: "Felicia" });
|
||||
assert.equal(db101.silenced, false);
|
||||
});
|
||||
|
||||
test("a server's own limit changes its guest's alert, and that server's maintenance silences it", async () => {
|
||||
const [s] = await db.insert(servers).values({ name: "weekly", apiTokenHash: hashToken("t"), apiTokenPrefix: "t", proxmoxIntegrationId: 1, proxmoxVmid: 102, thresholds: JSON.stringify({ backupStaleHours: 240 }) }).returning();
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
let alerts = await staleAlerts();
|
||||
assert.equal(alerts.some((a) => a.id.endsWith(":102")), false, "150 hours is fine for a limit of 240");
|
||||
await db.update(servers).set({ thresholds: JSON.stringify({ backupStaleHours: 100 }) });
|
||||
alerts = await staleAlerts();
|
||||
const weekly = alerts.find((a) => a.id.endsWith(":102"))!;
|
||||
assert.ok(weekly, "150 hours is over a limit of 100");
|
||||
assert.deepEqual(weekly.target && { type: weekly.target.type, id: weekly.target.id }, { type: "server", id: s.id }, "it is the server that can be silenced");
|
||||
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: s.id, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
|
||||
assert.equal((await staleAlerts()).find((a) => a.id.endsWith(":102"))!.silenced, true);
|
||||
await db.delete(maintenanceWindows);
|
||||
await db.update(servers).set({ thresholds: null });
|
||||
});
|
||||
|
||||
test("the limit in Settings is what the others follow, and 0 turns the alerts off", async () => {
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
await updateSettings({ healthChecks: { backupStaleHours: 100 } });
|
||||
assert.deepEqual((await staleAlerts()).map((a) => a.id.split(":").pop()).sort(), ["101", "102"], "60 hours is fine for a limit of 100");
|
||||
await updateSettings({ healthChecks: { backupStaleHours: 0 } });
|
||||
assert.deepEqual(await staleAlerts(), []);
|
||||
await updateSettings({ healthChecks: { backupStaleHours: 48 } });
|
||||
});
|
||||
|
||||
test("the daily list is the stale guests that aren't under maintenance", async () => {
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
const list = await collectStaleBackups();
|
||||
assert.equal(list.length, 3);
|
||||
await db.insert(maintenanceWindows).values({ targetType: "integration", targetId: 1, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
|
||||
assert.deepEqual(await collectStaleBackups(), [], "the whole Proxmox integration is being worked on");
|
||||
await db.delete(maintenanceWindows);
|
||||
await notifyStaleBackups([]); // nothing to say, nothing sent, and no error with no channel set up
|
||||
await notifyStaleBackups(list);
|
||||
});
|
||||
|
||||
test("/metrics says how many guests are in each state and how old each newest backup is, from what was last read", async () => {
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
invalidateAlertsCache();
|
||||
const text = await buildMetrics();
|
||||
assert.match(text, /homelab_backup_guests\{status="ok"\} 1/);
|
||||
assert.match(text, /homelab_backup_guests\{status="stale"\} 3/);
|
||||
const age = /homelab_backup_last_age_seconds\{guest="web",vmid="100",integration="Felicia"\} (\d+)/.exec(text);
|
||||
assert.ok(age, "an age for each guest with a backup");
|
||||
assert.ok(Math.abs(Number(age[1]) - 60 * 3600) < 120, `about 60 hours, got ${age[1]}`);
|
||||
// the page's own selector still asks for its own limit for every guest
|
||||
assert.equal((await getBackupCoverage({ staleHours: 24 * 14 })).counts.stale, 0);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user