Stale-backup alerts, per-server alert limits, and update runs from the Updates page

- Backups: a guest whose newest backup is older than its limit is an alert (backup kind),
  in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds)
- Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent,
  forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the
  global ones were; saving clears the alerts cache
- Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under
  maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and
  reports what is left and whether a restart is due; poller survives restarts
- Notification toggle for update-run results; tests and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 01:25:09 +02:00
1 parent 5dc6cea69c
commit 7a5b623444
42 files changed
+7561 -57

No files matched your search

+185
View File
@@ -0,0 +1,185 @@
/**
* Guests whose newest backup has gone too old: the limit (Settings, or a server's own), the alert, what silences it, the daily list and /metrics.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
const dir = mkdtempSync(join(tmpdir(), "hm-stale-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const { db } = await import("../src/db/client.js");
const { servers, maintenanceWindows, integrations } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { buildCoverage, staleBackups, collectStaleBackups, getBackupCoverage, setCoverageLoaderForTests } = await import("../src/services/backupCoverage.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { updateSettings } = await import("../src/services/settingsStore.js");
const { notifyStaleBackups } = await import("../src/services/notify.js");
const { hashToken } = await import("../src/services/tokens.js");
await runMigrations();
after(() => {
setCoverageLoaderForTests(null);
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const HOUR = 3600_000;
const guest = (vmid: number, name: string) => ({ vmid, name, node: "pve1", type: "qemu" as const, status: "running", cpu: null, maxmem: null, mem: null, uptime: null });
const allJob = { id: "j1", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null };
const px = (guests: ReturnType<typeof guest>[]) => [{ integrationId: 1, integrationName: "Felicia", guests, jobs: [allJob], tasks: [] }];
const snapAgo = (now: number, vmid: number, hours: number) => ({ backupType: "vm", backupId: String(vmid), backupTime: Math.floor((now - hours * HOUR) / 1000), verification: "ok" as const });
describe("when a guest is stale", () => {
const NOW = new Date("2026-10-17T12:00:00Z");
const now = NOW.getTime();
const input = (hoursAgo: number, extra: { servers?: Parameters<typeof buildCoverage>[0]["servers"]; staleHours?: number } = {}) => ({
proxmox: px([guest(100, "web")]),
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, hoursAgo)] }],
servers: extra.servers ?? [],
now: NOW,
staleHours: extra.staleHours,
});
test("a guest older than the limit is stale, and the limit it was judged by is on the row", () => {
const [g] = buildCoverage(input(60));
assert.equal(g.status, "stale");
assert.equal(g.staleAfterHours, 48);
assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].status, "ok");
assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].staleAfterHours, 72);
});
test("a server that is this guest can have its own limit — longer for a weekly backup, or 0 for never", () => {
const linked = (staleHours: number | null) => [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100, staleHours }];
assert.equal(buildCoverage(input(200, { servers: linked(240) }))[0].status, "ok", "eight days is fine for a limit of ten");
assert.equal(buildCoverage(input(300, { servers: linked(240) }))[0].status, "stale");
assert.equal(buildCoverage(input(5000, { servers: linked(0) }))[0].status, "ok", "0 means this one is never called stale");
assert.equal(buildCoverage(input(200, { servers: linked(null) }))[0].status, "stale", "no limit of its own: the global one");
// a failure is a failure whatever the limit says
const failed = buildCoverage({ ...input(1, { servers: linked(0) }), pbs: [{ integrationName: "PBS", groups: [{ ...snapAgo(now, 100, 1), verification: "failed" as const }] }] });
assert.equal(failed[0].status, "failed");
});
test("only stale guests are listed, oldest first, critical at three times the limit", () => {
const report = {
guests: buildCoverage({
proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "fresh"), guest(103, "never")]),
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 160), snapAgo(now, 102, 2)] }],
servers: [],
now: NOW,
}),
counts: { ok: 1, stale: 2, failed: 0, none: 0, unknown: 1, total: 4 },
sources: [],
staleHours: 48,
generatedAt: NOW.toISOString(),
};
const stale = staleBackups(report, NOW);
assert.deepEqual(stale.map((s) => [s.guest.name, s.severity]), [["db", "critical"], ["web", "warning"]]);
assert.equal(stale[0].key, "backup-stale:1:101");
assert.match(stale[0].message, /db \(#101\) \[Felicia\]: its newest backup is 6 days old \(the limit is 48 h\)/);
assert.match(stale[1].message, /60 h old \(the limit is 48 h\)/);
});
test("it is silenced by maintenance on the Proxmox integration or on the server the guest is", () => {
const base = { guests: buildCoverage({ ...input(60), servers: [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100 }] }), counts: { ok: 0, stale: 1, failed: 0, none: 0, unknown: 0, total: 1 }, sources: [], staleHours: 48, generatedAt: NOW.toISOString() };
assert.equal(staleBackups(base, NOW)[0].silenced, false);
assert.equal(staleBackups(base, NOW, new Set(["integration:1"]))[0].silenced, true);
assert.equal(staleBackups(base, NOW, new Set(["server:7"]))[0].silenced, true);
assert.equal(staleBackups(base, NOW, new Set(["server:8", "integration:2"]))[0].silenced, false);
});
});
describe("through the alerts, the daily list and /metrics", () => {
before(async () => {
// The guests are Proxmox's, read from a stand-in; the integration they belong to still has to exist for a server to be linked to one.
await db.insert(integrations).values({ id: 1, type: "proxmox", name: "Felicia", baseUrl: "https://pve.example", enabled: false });
});
// The alert pass judges by the real clock, so these backups are dated from it.
const fixture = () => {
const now = Date.now();
return {
proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "weekly"), guest(103, "fine")]).map((p) => ({ ...p })),
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 200), snapAgo(now, 102, 150), snapAgo(now, 103, 3)] }],
sources: [{ kind: "proxmox" as const, name: "Felicia", ok: true }, { kind: "pbs" as const, name: "PBS", ok: true }],
};
};
const staleAlerts = async () => {
invalidateAlertsCache();
return (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:backup-stale:"));
};
test("each stale guest is an alert of the backup kind, with a link, a target and a severity", async () => {
setCoverageLoaderForTests(async () => fixture());
const alerts = await staleAlerts();
assert.deepEqual(alerts.map((a) => a.id).sort(), ["backup:backup-stale:1:100", "backup:backup-stale:1:101", "backup:backup-stale:1:102"]);
const db101 = alerts.find((a) => a.id.endsWith(":101"))!;
assert.equal(db101.category, "backup");
assert.equal(db101.severity, "critical", "200 hours is more than three times 48");
assert.equal(alerts.find((a) => a.id.endsWith(":100"))!.severity, "warning");
assert.equal(db101.link, "/backups");
assert.deepEqual(db101.target, { type: "integration", id: 1, name: "Felicia" });
assert.equal(db101.silenced, false);
});
test("a server's own limit changes its guest's alert, and that server's maintenance silences it", async () => {
const [s] = await db.insert(servers).values({ name: "weekly", apiTokenHash: hashToken("t"), apiTokenPrefix: "t", proxmoxIntegrationId: 1, proxmoxVmid: 102, thresholds: JSON.stringify({ backupStaleHours: 240 }) }).returning();
setCoverageLoaderForTests(async () => fixture());
let alerts = await staleAlerts();
assert.equal(alerts.some((a) => a.id.endsWith(":102")), false, "150 hours is fine for a limit of 240");
await db.update(servers).set({ thresholds: JSON.stringify({ backupStaleHours: 100 }) });
alerts = await staleAlerts();
const weekly = alerts.find((a) => a.id.endsWith(":102"))!;
assert.ok(weekly, "150 hours is over a limit of 100");
assert.deepEqual(weekly.target && { type: weekly.target.type, id: weekly.target.id }, { type: "server", id: s.id }, "it is the server that can be silenced");
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: s.id, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
assert.equal((await staleAlerts()).find((a) => a.id.endsWith(":102"))!.silenced, true);
await db.delete(maintenanceWindows);
await db.update(servers).set({ thresholds: null });
});
test("the limit in Settings is what the others follow, and 0 turns the alerts off", async () => {
setCoverageLoaderForTests(async () => fixture());
await updateSettings({ healthChecks: { backupStaleHours: 100 } });
assert.deepEqual((await staleAlerts()).map((a) => a.id.split(":").pop()).sort(), ["101", "102"], "60 hours is fine for a limit of 100");
await updateSettings({ healthChecks: { backupStaleHours: 0 } });
assert.deepEqual(await staleAlerts(), []);
await updateSettings({ healthChecks: { backupStaleHours: 48 } });
});
test("the daily list is the stale guests that aren't under maintenance", async () => {
setCoverageLoaderForTests(async () => fixture());
const list = await collectStaleBackups();
assert.equal(list.length, 3);
await db.insert(maintenanceWindows).values({ targetType: "integration", targetId: 1, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
assert.deepEqual(await collectStaleBackups(), [], "the whole Proxmox integration is being worked on");
await db.delete(maintenanceWindows);
await notifyStaleBackups([]); // nothing to say, nothing sent, and no error with no channel set up
await notifyStaleBackups(list);
});
test("/metrics says how many guests are in each state and how old each newest backup is, from what was last read", async () => {
setCoverageLoaderForTests(async () => fixture());
invalidateAlertsCache();
const text = await buildMetrics();
assert.match(text, /homelab_backup_guests\{status="ok"\} 1/);
assert.match(text, /homelab_backup_guests\{status="stale"\} 3/);
const age = /homelab_backup_last_age_seconds\{guest="web",vmid="100",integration="Felicia"\} (\d+)/.exec(text);
assert.ok(age, "an age for each guest with a backup");
assert.ok(Math.abs(Number(age[1]) - 60 * 3600) < 120, `about 60 hours, got ${age[1]}`);
// the page's own selector still asks for its own limit for every guest
assert.equal((await getBackupCoverage({ staleHours: 24 * 14 })).counts.stale, 0);
});
});