Endpoint checks, restore tests, and a change log
- Endpoints: HTTP, TCP and TLS checks with answer time, uptime and certificate expiry; failures and expiring certificates become alerts and notifications; linked to a server for maintenance and tags - Restore tests: record when a backup was restored and how it went; failed tests always alert, overdue ones with a limit set; shown on the Backup coverage page - Change log: one filterable timeline made from the audit log and the incident record Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c3bc6bbf72
commit
92726568b7
37 files changed
+6918
-21
No files matched your search
@@ -0,0 +1,210 @@
|
||||
/**
|
||||
* Restore tests: what the newest test of something amounts to, when to speak up, and the routes that record them.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-restore-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, restoreTests, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { restoreInfo, restoreAlerts, customKey, guestKey } = await import("../src/services/restoreTests.js");
|
||||
const { setCoverageLoaderForTests, buildCoverage } = await import("../src/services/backupCoverage.js");
|
||||
const { backupsRouter } = await import("../src/routes/backups.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const NOW = new Date("2026-10-18T12:00:00Z");
|
||||
const daysAgo = (n: number) => new Date(NOW.getTime() - n * 86_400_000).toISOString().slice(0, 10);
|
||||
const test1 = (result: string, ago: number, extra: Record<string, unknown> = {}) => ({ result, testedAt: daysAgo(ago), note: null as string | null, testedBy: "Ada", ...extra });
|
||||
|
||||
describe("what the newest test amounts to", () => {
|
||||
test("how it went; none at all is never; an old pass is stale once there is a limit; a failure or 'not needed' doesn't age", () => {
|
||||
assert.equal(restoreInfo(undefined, NOW, 90).status, "never");
|
||||
assert.deepEqual(restoreInfo(test1("ok", 10), NOW, 90), { status: "ok", testedAt: daysAgo(10), ageDays: 10, note: null, testedBy: "Ada" });
|
||||
assert.equal(restoreInfo(test1("ok", 100), NOW, 90).status, "stale");
|
||||
assert.equal(restoreInfo(test1("ok", 100), NOW, 0).status, "ok", "no limit, no staleness");
|
||||
assert.equal(restoreInfo(test1("partial", 100), NOW, 90).status, "stale");
|
||||
assert.equal(restoreInfo(test1("failed", 400), NOW, 90).status, "failed");
|
||||
assert.equal(restoreInfo(test1("not_needed", 400), NOW, 90).status, "not_needed");
|
||||
});
|
||||
|
||||
test("something that isn't a guest is told apart by its name", () => {
|
||||
assert.equal(customKey("Photos share"), "custom:photos-share");
|
||||
assert.equal(customKey(" Gitea — database! "), "custom:gitea-database");
|
||||
assert.equal(customKey("Åkerblom's NAS"), "custom:akerblom-s-nas");
|
||||
assert.equal(customKey("!!!"), null);
|
||||
assert.equal(guestKey(2, 105), "guest:2:105");
|
||||
});
|
||||
});
|
||||
|
||||
const guest = (vmid: number, name: string) => ({ vmid, name, node: "pve1", type: "qemu" as const, status: "running", cpu: null, maxmem: null, mem: null, uptime: null });
|
||||
const fixture = () => {
|
||||
const loaded = {
|
||||
proxmox: [{ integrationId: 1, integrationName: "Felicia", guests: [guest(100, "elsa"), guest(101, "anna"), guest(102, "zoe")], jobs: [{ id: "j", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null }], tasks: [] }],
|
||||
pbs: [{ integrationName: "Backup server", groups: [{ backupType: "vm", backupId: "100", backupTime: Math.floor(Date.now() / 1000) - 3600, verification: "ok" as const }, { backupType: "vm", backupId: "101", backupTime: Math.floor(Date.now() / 1000) - 7200, verification: "ok" as const }] }],
|
||||
sources: [{ kind: "proxmox" as const, name: "Felicia", ok: true }],
|
||||
};
|
||||
return loaded;
|
||||
};
|
||||
|
||||
describe("when to speak up", () => {
|
||||
const report = () => ({ guests: buildCoverage({ ...fixture(), servers: [], now: NOW }), counts: {} as never, sources: [], staleHours: 48, generatedAt: NOW.toISOString() });
|
||||
const row = (key: string, label: string, result: string, ago: number) => [key, { id: 1, subjectKey: key, label, testedAt: daysAgo(ago), result, note: null, testedBy: null, createdAt: "" }] as const;
|
||||
|
||||
test("with no limit only a failure speaks; with one, a guest that has backups and no recent test does too", () => {
|
||||
const g = report();
|
||||
assert.deepEqual(restoreAlerts(g, new Map(), 0, NOW), [], "no limit, nothing tested: quiet");
|
||||
const limited = restoreAlerts(g, new Map(), 90, NOW);
|
||||
assert.deepEqual(limited.map((a) => [a.guest?.name, a.severity]), [["elsa", "warning"], ["anna", "warning"]], "zoe has no backups to restore from, so nothing to say");
|
||||
assert.match(limited[0].message, /elsa \(#100\) \[Felicia\]: a restore has never been tested/);
|
||||
const failed = restoreAlerts(g, new Map([row("guest:1:100", "elsa", "failed", 3)]), 0, NOW);
|
||||
assert.equal(failed.length, 1);
|
||||
assert.equal(failed[0].severity, "critical");
|
||||
assert.match(failed[0].message, /the last restore test \(.*\) failed/);
|
||||
});
|
||||
|
||||
test("a recent pass, 'not needed' and an old pass under the limit are all fine; an old one isn't", () => {
|
||||
const g = report();
|
||||
const tests = new Map([row("guest:1:100", "elsa", "ok", 10), row("guest:1:101", "anna", "not_needed", 500)]);
|
||||
assert.deepEqual(restoreAlerts(g, tests, 90, NOW), []);
|
||||
const old = restoreAlerts(g, new Map([row("guest:1:100", "elsa", "ok", 120), row("guest:1:101", "anna", "ok", 1)]), 90, NOW);
|
||||
assert.equal(old.length, 1);
|
||||
assert.match(old[0].message, /last restore test was 120 days ago \(the limit is 90 days\)/);
|
||||
});
|
||||
|
||||
test("something that isn't a guest is told of when it failed, or when it is stale", () => {
|
||||
const g = report();
|
||||
const tests = new Map([row("custom:photos", "Photos share", "ok", 200), row("custom:db", "Gitea database", "failed", 5), row("custom:fine", "Fine", "ok", 3), row("guest:9:9", "gone guest", "ok", 999)]);
|
||||
const out = restoreAlerts(g, tests, 90, NOW);
|
||||
assert.deepEqual(out.map((a) => a.message.split(":")[0]).sort(), ["Gitea database", "Photos share", "anna (#101) [Felicia]", "elsa (#100) [Felicia]"].sort());
|
||||
assert.equal(restoreAlerts(g, tests, 0, NOW).filter((a) => !a.guest).length, 1, "with no limit only the failure");
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes, the page, the alerts and the figures", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/backups", backupsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/backups`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
setCoverageLoaderForTests(null);
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
|
||||
test("an operator records a test; a viewer can't; a wrong one is refused", async () => {
|
||||
assert.equal((await send("POST", "/restore-tests", { key: "guest:1:100", label: "elsa", result: "ok", note: "booted and the data was there" })).status, 201);
|
||||
who = "viewer";
|
||||
assert.equal((await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "ok" })).status, 403);
|
||||
who = "operator";
|
||||
for (const bad of [
|
||||
{ key: "guest:1:101", label: "anna", result: "excellent" },
|
||||
{ key: "guest:one:two", label: "anna", result: "ok" },
|
||||
{ key: "../etc", label: "anna", result: "ok" },
|
||||
{ key: "guest:1:101", label: "anna", result: "ok", testedAt: "2999-01-01" },
|
||||
{ key: "guest:1:101", label: "anna", result: "ok", testedAt: "2026-02-30" },
|
||||
{ label: "!!!", result: "ok" },
|
||||
{ key: "guest:1:101", label: "", result: "ok" },
|
||||
]) assert.equal((await send("POST", "/restore-tests", bad)).status, 400, JSON.stringify(bad));
|
||||
assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "restore_test" && e.action === "record").length, 1);
|
||||
});
|
||||
|
||||
test("the Backups page's data says how each guest's restore went, or that nobody has tried", async () => {
|
||||
const cov = await (await send("GET", "/coverage")).json();
|
||||
const by = Object.fromEntries(cov.guests.map((g: { name: string }) => [g.name, g]));
|
||||
assert.equal(by.elsa.restoreKey, "guest:1:100");
|
||||
assert.deepEqual([by.elsa.restoreTest.status, by.elsa.restoreTest.testedAt, by.elsa.restoreTest.note], ["ok", today, "booted and the data was there"]);
|
||||
assert.equal(by.anna.restoreTest.status, "never");
|
||||
assert.equal(cov.restoreTestDays, 0);
|
||||
});
|
||||
|
||||
test("something that isn't a guest gets a key from its name, and is listed with how it stands", async () => {
|
||||
const res = await send("POST", "/restore-tests", { label: "Photos share", result: "ok", testedAt: daysAgo(40) });
|
||||
assert.equal(res.status, 201);
|
||||
assert.equal((await res.json()).test.subjectKey, "custom:photos-share");
|
||||
const list = await (await send("GET", "/restore-tests")).json();
|
||||
assert.deepEqual(list.custom.map((c: { key: string; status: string }) => [c.key, c.status]), [["custom:photos-share", "ok"]]);
|
||||
assert.equal(list.tests.length, 2);
|
||||
});
|
||||
|
||||
test("a failed test is an alert straight away; with a limit, never-tested and old ones are too; marking 'not needed' ends it", async () => {
|
||||
await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "failed", note: "the disk wouldn't attach" });
|
||||
invalidateAlertsCache();
|
||||
let alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:"));
|
||||
assert.deepEqual(alerts.map((a) => [a.id, a.severity]), [["backup:restore-test:guest:1:101", "critical"]]);
|
||||
assert.match(alerts[0].message, /anna \(#101\).*failed — the disk wouldn't attach/);
|
||||
assert.equal(alerts[0].link, "/backups");
|
||||
|
||||
await updateSettings({ healthChecks: { restoreTestDays: 30 } });
|
||||
invalidateAlertsCache();
|
||||
alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:"));
|
||||
assert.deepEqual(alerts.map((a) => a.id).sort(), ["backup:restore-test:custom:photos-share", "backup:restore-test:guest:1:101"], "elsa is fine; the photos share was last tested 40 days ago");
|
||||
|
||||
await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "not_needed", note: "a scratch machine" });
|
||||
invalidateAlertsCache();
|
||||
alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:"));
|
||||
assert.deepEqual(alerts.map((a) => a.id), ["backup:restore-test:custom:photos-share"]);
|
||||
await updateSettings({ healthChecks: { restoreTestDays: 0 } });
|
||||
});
|
||||
|
||||
test("metrics say how old each test is and whether it passed", async () => {
|
||||
const text = await buildMetrics();
|
||||
assert.match(text, /homelab_restore_test_age_days\{subject="elsa"\} 0/);
|
||||
assert.match(text, /homelab_restore_test_age_days\{subject="Photos share"\} 40/);
|
||||
assert.match(text, /homelab_restore_test_passed\{subject="elsa"\} 1/);
|
||||
assert.doesNotMatch(text, /subject="anna"/, "'not needed' isn't measured");
|
||||
});
|
||||
|
||||
test("only an administrator takes an entry out, and that is recorded", async () => {
|
||||
const [first] = await db.select().from(restoreTests);
|
||||
assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 403);
|
||||
who = "admin";
|
||||
assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 204);
|
||||
assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 404);
|
||||
assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "restore_test" && e.action === "delete").length, 1);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user