- Runbooks: notes per server/integration, shown next to matching alerts - Backup coverage page across Proxmox and PBS, with a digest section - Hardware inventory with warranty, power and running cost - Authentik group to role mapping with preview and safeguards - /metrics (bearer token) and signed outgoing event webhooks - Docs: README, DATABASE, NOTIFICATIONS, ROLES and new API.md Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
175 lines
9.3 KiB
TypeScript
175 lines
9.3 KiB
TypeScript
/**
|
|
* Backup coverage: what the rows say for each kind of guest, and the route and the digest around them.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { AddressInfo } from "node:net";
|
|
import type { Server } from "node:http";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-coverage-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const express = (await import("express")).default;
|
|
const { db } = await import("../src/db/client.js");
|
|
const { users, servers } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const { buildCoverage, summarize, setCoverageLoaderForTests } = await import("../src/services/backupCoverage.js");
|
|
const { backupsRouter } = await import("../src/routes/backups.js");
|
|
const { buildDigest } = await import("../src/services/weeklyDigest.js");
|
|
|
|
await runMigrations();
|
|
|
|
const NOW = new Date("2026-10-17T12:00:00Z");
|
|
const HOUR = 3600_000;
|
|
const guest = (vmid: number, name: string, type: "qemu" | "lxc" = "qemu", node = "pve1") => ({ vmid, name, node, type, status: "running", cpu: null, maxmem: null, mem: null, uptime: null });
|
|
const allJob = { id: "j1", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null };
|
|
const task = (guestId: string | null, ago: number, ok = true, status = ok ? "OK" : "job errors") => ({
|
|
node: "pve1",
|
|
upid: `upid-${guestId}-${ago}`,
|
|
guestId,
|
|
status,
|
|
ok,
|
|
startTime: new Date(NOW.getTime() - ago * HOUR).toISOString(),
|
|
endTime: new Date(NOW.getTime() - ago * HOUR + 600_000).toISOString(),
|
|
});
|
|
const snap = (type: string, id: string, ago: number, verification: "ok" | "failed" | null = "ok") => ({ backupType: type, backupId: id, backupTime: Math.floor((NOW.getTime() - ago * HOUR) / 1000), verification });
|
|
|
|
const px = (guests: ReturnType<typeof guest>[], jobs: (typeof allJob)[], tasks: ReturnType<typeof task>[]) => [{ integrationId: 1, integrationName: "Felicia", guests, jobs, tasks }];
|
|
|
|
describe("what each guest reads as", () => {
|
|
test("backed up recently to a Backup Server: ok; a while ago: stale", () => {
|
|
const rows = buildCoverage({ proxmox: px([guest(100, "web"), guest(101, "db", "lxc")], [allJob], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 5), snap("ct", "101", 100)] }], servers: [], now: NOW });
|
|
const by = Object.fromEntries(rows.map((r) => [r.name, r]));
|
|
assert.equal(by.web.status, "ok");
|
|
assert.equal(by.web.lastSource, "pbs");
|
|
assert.equal(by.db.status, "stale", "100 hours is more than the 48 by default");
|
|
assert.equal(by.web.lastBackupAt, new Date(NOW.getTime() - 5 * HOUR).toISOString());
|
|
});
|
|
|
|
test("a task that names the guest counts, and the newer of the two sources wins", () => {
|
|
const rows = buildCoverage({ proxmox: px([guest(100, "web")], [allJob], [task("100", 30), task("100", 3, false)]), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 10)] }], servers: [], now: NOW });
|
|
assert.equal(rows[0].lastSource, "pbs", "the snapshot (10 h) is newer than the successful task (30 h)");
|
|
assert.equal(rows[0].status, "failed", "but the latest run, 3 h ago, failed");
|
|
assert.match(rows[0].failure!, /job errors/);
|
|
});
|
|
|
|
test("a failed run followed by a good backup is not a failure any more", () => {
|
|
const rows = buildCoverage({ proxmox: px([guest(100, "web")], [allJob], [task("100", 8, false), task("100", 2, true)]), pbs: [], servers: [], now: NOW });
|
|
assert.equal(rows[0].status, "ok");
|
|
assert.equal(rows[0].lastSource, "task");
|
|
});
|
|
|
|
test("a snapshot that failed verification is a failure", () => {
|
|
const rows = buildCoverage({ proxmox: px([guest(100, "web")], [allJob], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 2, "failed")] }], servers: [], now: NOW });
|
|
assert.equal(rows[0].status, "failed");
|
|
assert.match(rows[0].failure!, /verification/);
|
|
});
|
|
|
|
test("in a job with nothing to say when it last ran is unknown; in no job and no backup is none", () => {
|
|
const only101 = { ...allJob, allGuests: false, vmids: "100" };
|
|
const rows = buildCoverage({ proxmox: px([guest(100, "covered"), guest(101, "forgotten")], [only101], []), pbs: [], servers: [], now: NOW });
|
|
const by = Object.fromEntries(rows.map((r) => [r.name, r]));
|
|
assert.equal(by.covered.status, "unknown");
|
|
assert.equal(by.covered.inJob, true);
|
|
assert.equal(by.forgotten.status, "none");
|
|
assert.equal(by.forgotten.inJob, false);
|
|
});
|
|
|
|
test("a guest in no job that still has an old backup is judged by its age", () => {
|
|
const rows = buildCoverage({ proxmox: px([guest(101, "manual")], [], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "101", 6)] }], servers: [], now: NOW });
|
|
assert.equal(rows[0].status, "ok");
|
|
assert.equal(rows[0].inJob, false);
|
|
});
|
|
|
|
test("how long counts as stale is up to the caller", () => {
|
|
const input = { proxmox: px([guest(100, "web")], [allJob], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 100)] }], servers: [], now: NOW };
|
|
assert.equal(buildCoverage({ ...input, staleHours: 24 * 3 })[0].status, "stale");
|
|
assert.equal(buildCoverage({ ...input, staleHours: 24 * 7 })[0].status, "ok");
|
|
});
|
|
|
|
test("a VM and a container with the same number aren't mixed up, and a linked server is named", () => {
|
|
const rows = buildCoverage({
|
|
proxmox: px([guest(100, "vm100", "qemu"), guest(100, "ct100", "lxc", "pve2")], [allJob], []),
|
|
pbs: [{ integrationName: "PBS", groups: [snap("ct", "100", 1)] }],
|
|
servers: [{ id: 7, name: "elsa", proxmoxIntegrationId: 1, proxmoxVmid: 100 }],
|
|
now: NOW,
|
|
});
|
|
const by = Object.fromEntries(rows.map((r) => [r.name, r]));
|
|
assert.equal(by.ct100.status, "ok");
|
|
assert.equal(by.vm100.status, "unknown", "only the container has a snapshot");
|
|
assert.deepEqual(by.vm100.server, { id: 7, name: "elsa" });
|
|
});
|
|
|
|
test("the counts add up", () => {
|
|
const rows = buildCoverage({ proxmox: px([guest(100, "a"), guest(101, "b")], [{ ...allJob, allGuests: false, vmids: "100" }], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 1)] }], servers: [], now: NOW });
|
|
assert.deepEqual(summarize(rows), { ok: 1, stale: 0, failed: 0, none: 1, unknown: 0, total: 2 });
|
|
});
|
|
});
|
|
|
|
describe("the route and the digest", () => {
|
|
let app: Server;
|
|
let base = "";
|
|
|
|
before(async () => {
|
|
await db.insert(users).values({ oidcSub: "viewer", name: "Vic", role: "viewer" });
|
|
await db.insert(servers).values({ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", lastSeenAt: new Date().toISOString() });
|
|
setCoverageLoaderForTests(async () => ({
|
|
proxmox: px([guest(100, "web"), guest(101, "lonely")], [{ ...allJob, allGuests: false, vmids: "100" }], []).map((p) => ({ ...p })),
|
|
pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 100)] }],
|
|
sources: [
|
|
{ kind: "proxmox" as const, name: "Felicia", ok: true },
|
|
{ kind: "pbs" as const, name: "PBS", ok: false, error: "couldn't read tank" },
|
|
],
|
|
}));
|
|
const a = express();
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
|
next();
|
|
});
|
|
a.use("/api/backups", backupsRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/backups`;
|
|
});
|
|
|
|
after(() => {
|
|
setCoverageLoaderForTests(null);
|
|
app.close();
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows keeps the database file locked until the process exits.
|
|
}
|
|
});
|
|
|
|
test("anyone signed in gets the rows, the counts, and which sources couldn't be read", async () => {
|
|
const body = await (await fetch(`${base}/coverage`)).json();
|
|
assert.equal(body.staleHours, 48);
|
|
assert.deepEqual(body.guests.map((g: { name: string }) => g.name), ["lonely", "web"]);
|
|
assert.equal(body.counts.total, 2);
|
|
assert.equal(body.sources.find((s: { name: string }) => s.name === "PBS").ok, false);
|
|
});
|
|
|
|
test("the stale threshold is set in days, and nonsense falls back to the default", async () => {
|
|
// web's newest snapshot is 100 hours old: stale at 2 days, fine at 7.
|
|
assert.equal((await (await fetch(`${base}/coverage?staleDays=7`)).json()).guests.find((g: { name: string }) => g.name === "web").status, "ok");
|
|
assert.equal((await (await fetch(`${base}/coverage?staleDays=2`)).json()).guests.find((g: { name: string }) => g.name === "web").status, "stale");
|
|
assert.equal((await (await fetch(`${base}/coverage?staleDays=banana`)).json()).staleHours, 48);
|
|
assert.equal((await (await fetch(`${base}/coverage?staleDays=9999`)).json()).staleHours, 48);
|
|
});
|
|
|
|
test("the weekly digest lists what isn't backed up, worst first", async () => {
|
|
const digest = await buildDigest(new Date());
|
|
assert.match(digest.message, /BACKUPS\n0 of 2 guests have a recent backup\.\n- lonely \(VM 101\): no backup, and not in any backup job\n- web \(VM 100\): last backup \d+ d( \d+ h)? ago/);
|
|
});
|
|
});
|