Files
Homelab-manager/server/test/backupCoverage.test.ts
T
bobbanandClaude Sonnet 5.5 5e26e77340 UniFi: servers plugged into switches; server page in sections; My work; settings search and undo
UniFi: the controller's client list is matched to servers by IP address, so a
server is connected through a switch or access point (and port). That is a
dependency made by itself, shown on the server page, the Network devices page
and the Connections map. IP Addresses can sync the devices' addresses.

A server's page is split into Overview, Health, Network, History, Tasks and
Settings, by ?view=; alerts link to the section they are about.

My work lists what has your name on it: alerts you have, maintenance you
started (extend, end with undo) and update runs you started, with a badge.

Settings has a search box (and the command palette lists every setting).
Removing a dependency, admin link, runbook, hand-made IP address, port opening
or restore test is undone from the message instead of asked about first.

Two backup tests used a fixed date and failed as days passed; they follow the
real clock now.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-20 19:35:45 +02:00

176 lines
9.5 KiB
TypeScript

/**
* Backup coverage: what the rows say for each kind of guest, and the route and the digest around them.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-coverage-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { buildCoverage, summarize, setCoverageLoaderForTests } = await import("../src/services/backupCoverage.js");
const { backupsRouter } = await import("../src/routes/backups.js");
const { buildDigest } = await import("../src/services/weeklyDigest.js");
await runMigrations();
// The routes read the real clock, so the fixtures are laid out from it: a fixed date made these tests fail once enough days had passed.
const NOW = new Date(Math.floor(Date.now() / 1000) * 1000); // whole seconds, as snapshot times are
const HOUR = 3600_000;
const guest = (vmid: number, name: string, type: "qemu" | "lxc" = "qemu", node = "pve1") => ({ vmid, name, node, type, status: "running", cpu: null, maxmem: null, mem: null, uptime: null });
const allJob = { id: "j1", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null };
const task = (guestId: string | null, ago: number, ok = true, status = ok ? "OK" : "job errors") => ({
node: "pve1",
upid: `upid-${guestId}-${ago}`,
guestId,
status,
ok,
startTime: new Date(NOW.getTime() - ago * HOUR).toISOString(),
endTime: new Date(NOW.getTime() - ago * HOUR + 600_000).toISOString(),
});
const snap = (type: string, id: string, ago: number, verification: "ok" | "failed" | null = "ok") => ({ backupType: type, backupId: id, backupTime: Math.floor((NOW.getTime() - ago * HOUR) / 1000), verification });
const px = (guests: ReturnType<typeof guest>[], jobs: (typeof allJob)[], tasks: ReturnType<typeof task>[]) => [{ integrationId: 1, integrationName: "Felicia", guests, jobs, tasks }];
describe("what each guest reads as", () => {
test("backed up recently to a Backup Server: ok; a while ago: stale", () => {
const rows = buildCoverage({ proxmox: px([guest(100, "web"), guest(101, "db", "lxc")], [allJob], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 5), snap("ct", "101", 100)] }], servers: [], now: NOW });
const by = Object.fromEntries(rows.map((r) => [r.name, r]));
assert.equal(by.web.status, "ok");
assert.equal(by.web.lastSource, "pbs");
assert.equal(by.db.status, "stale", "100 hours is more than the 48 by default");
assert.equal(by.web.lastBackupAt, new Date(NOW.getTime() - 5 * HOUR).toISOString());
});
test("a task that names the guest counts, and the newer of the two sources wins", () => {
const rows = buildCoverage({ proxmox: px([guest(100, "web")], [allJob], [task("100", 30), task("100", 3, false)]), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 10)] }], servers: [], now: NOW });
assert.equal(rows[0].lastSource, "pbs", "the snapshot (10 h) is newer than the successful task (30 h)");
assert.equal(rows[0].status, "failed", "but the latest run, 3 h ago, failed");
assert.match(rows[0].failure!, /job errors/);
});
test("a failed run followed by a good backup is not a failure any more", () => {
const rows = buildCoverage({ proxmox: px([guest(100, "web")], [allJob], [task("100", 8, false), task("100", 2, true)]), pbs: [], servers: [], now: NOW });
assert.equal(rows[0].status, "ok");
assert.equal(rows[0].lastSource, "task");
});
test("a snapshot that failed verification is a failure", () => {
const rows = buildCoverage({ proxmox: px([guest(100, "web")], [allJob], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 2, "failed")] }], servers: [], now: NOW });
assert.equal(rows[0].status, "failed");
assert.match(rows[0].failure!, /verification/);
});
test("in a job with nothing to say when it last ran is unknown; in no job and no backup is none", () => {
const only101 = { ...allJob, allGuests: false, vmids: "100" };
const rows = buildCoverage({ proxmox: px([guest(100, "covered"), guest(101, "forgotten")], [only101], []), pbs: [], servers: [], now: NOW });
const by = Object.fromEntries(rows.map((r) => [r.name, r]));
assert.equal(by.covered.status, "unknown");
assert.equal(by.covered.inJob, true);
assert.equal(by.forgotten.status, "none");
assert.equal(by.forgotten.inJob, false);
});
test("a guest in no job that still has an old backup is judged by its age", () => {
const rows = buildCoverage({ proxmox: px([guest(101, "manual")], [], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "101", 6)] }], servers: [], now: NOW });
assert.equal(rows[0].status, "ok");
assert.equal(rows[0].inJob, false);
});
test("how long counts as stale is up to the caller", () => {
const input = { proxmox: px([guest(100, "web")], [allJob], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 100)] }], servers: [], now: NOW };
assert.equal(buildCoverage({ ...input, staleHours: 24 * 3 })[0].status, "stale");
assert.equal(buildCoverage({ ...input, staleHours: 24 * 7 })[0].status, "ok");
});
test("a VM and a container with the same number aren't mixed up, and a linked server is named", () => {
const rows = buildCoverage({
proxmox: px([guest(100, "vm100", "qemu"), guest(100, "ct100", "lxc", "pve2")], [allJob], []),
pbs: [{ integrationName: "PBS", groups: [snap("ct", "100", 1)] }],
servers: [{ id: 7, name: "elsa", proxmoxIntegrationId: 1, proxmoxVmid: 100 }],
now: NOW,
});
const by = Object.fromEntries(rows.map((r) => [r.name, r]));
assert.equal(by.ct100.status, "ok");
assert.equal(by.vm100.status, "unknown", "only the container has a snapshot");
assert.deepEqual(by.vm100.server, { id: 7, name: "elsa" });
});
test("the counts add up", () => {
const rows = buildCoverage({ proxmox: px([guest(100, "a"), guest(101, "b")], [{ ...allJob, allGuests: false, vmids: "100" }], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 1)] }], servers: [], now: NOW });
assert.deepEqual(summarize(rows), { ok: 1, stale: 0, failed: 0, none: 1, unknown: 0, total: 2 });
});
});
describe("the route and the digest", () => {
let app: Server;
let base = "";
before(async () => {
await db.insert(users).values({ oidcSub: "viewer", name: "Vic", role: "viewer" });
await db.insert(servers).values({ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", lastSeenAt: new Date().toISOString() });
setCoverageLoaderForTests(async () => ({
proxmox: px([guest(100, "web"), guest(101, "lonely")], [{ ...allJob, allGuests: false, vmids: "100" }], []).map((p) => ({ ...p })),
pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 100)] }],
sources: [
{ kind: "proxmox" as const, name: "Felicia", ok: true },
{ kind: "pbs" as const, name: "PBS", ok: false, error: "couldn't read tank" },
],
}));
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/backups", backupsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/backups`;
});
after(() => {
setCoverageLoaderForTests(null);
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("anyone signed in gets the rows, the counts, and which sources couldn't be read", async () => {
const body = await (await fetch(`${base}/coverage`)).json();
assert.equal(body.staleHours, 48);
assert.deepEqual(body.guests.map((g: { name: string }) => g.name), ["lonely", "web"]);
assert.equal(body.counts.total, 2);
assert.equal(body.sources.find((s: { name: string }) => s.name === "PBS").ok, false);
});
test("the stale threshold is set in days, and nonsense falls back to the default", async () => {
// web's newest snapshot is 100 hours old: stale at 2 days, fine at 7.
assert.equal((await (await fetch(`${base}/coverage?staleDays=7`)).json()).guests.find((g: { name: string }) => g.name === "web").status, "ok");
assert.equal((await (await fetch(`${base}/coverage?staleDays=2`)).json()).guests.find((g: { name: string }) => g.name === "web").status, "stale");
assert.equal((await (await fetch(`${base}/coverage?staleDays=banana`)).json()).staleHours, 48);
assert.equal((await (await fetch(`${base}/coverage?staleDays=9999`)).json()).staleHours, 48);
});
test("the weekly digest lists what isn't backed up, worst first", async () => {
const digest = await buildDigest(new Date());
assert.match(digest.message, /BACKUPS\n0 of 2 guests have a recent backup\.\n- lonely \(VM 101\): no backup, and not in any backup job\n- web \(VM 100\): last backup \d+ d( \d+ h)? ago/);
});
});