Endpoint checks, restore tests, and a change log

- Endpoints: HTTP, TCP and TLS checks with answer time, uptime and certificate expiry; failures and expiring certificates become alerts and notifications; linked to a server for maintenance and tags
- Restore tests: record when a backup was restored and how it went; failed tests always alert, overdue ones with a limit set; shown on the Backup coverage page
- Change log: one filterable timeline made from the audit log and the incident record

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 23:03:56 +02:00
1 parent c3bc6bbf72
commit 92726568b7
37 files changed
+6918 -21

No files matched your search

+162
View File
@@ -0,0 +1,162 @@
/**
* The change log: the sentences, what is left out, the filters (kind, server, time, a word), paging, and who may read it.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-changes-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, auditLog, alertIncidents } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { describeChange, getChangeLog } = await import("../src/services/changeLog.js");
const { changesRouter } = await import("../src/routes/changes.js");
await runMigrations();
describe("the sentences", () => {
test("maintenance, dependencies, hardware, restore tests and settings read as sentences", () => {
assert.deepEqual(describeChange("maintenance", "start", { name: "nas", minutes: 60, reason: "reboot" }), { title: "Maintenance started on nas", tone: "info", detail: "for 60 min · reboot" });
assert.equal(describeChange("maintenance", "extend", { name: "nas", minutes: 240 }).detail, "for 4 h from now");
assert.equal(describeChange("dependency", "create", { from: "Gitea", to: "nas" }).title, "Gitea now needs nas");
assert.equal(describeChange("dependency", "delete", { from: "Gitea", to: "nas" }).title, "Gitea no longer needs nas");
assert.equal(describeChange("inventory", "create", { name: "NAS", kind: "storage" }).title, "Hardware added: NAS");
assert.deepEqual(describeChange("restore_test", "record", { name: "elsa", result: "failed", testedAt: "2026-10-01", note: "no boot" }), { title: "Restore test of elsa failed", tone: "bad", detail: "2026-10-01 · no boot" });
assert.equal(describeChange("restore_test", "record", { name: "elsa", result: "ok" }).tone, "good");
const s = describeChange("settings", "update", { sections: ["notifications", "webhook"], changes: { notifications: { healthAlerts: { from: true, to: false } }, webhook: { url: "(changed)" } } });
assert.equal(s.title, "Settings changed: notifications, webhook");
assert.equal(s.detail, 'notifications.healthAlerts: true → false · webhook.url changed', "a secret's value is never in it");
});
test("a server's own entries use the wording of its Activity card; anything else is made readable", () => {
assert.equal(describeChange("server", "agent_updated", { from: 4, to: 6 }).title, "Agent updated to version 6");
assert.equal(describeChange("alert", "ack", { assignee: "Olle", hours: 4 }).title, "Acknowledged, handed to Olle");
assert.deepEqual(describeChange("dns", "add_record", { name: "a.example.com", type: "A" }), { title: "Dns: add record — a.example.com", tone: "neutral", detail: "type: A" });
});
});
describe("through the routes", () => {
let app: Server;
let base = "";
let who = "operator";
let nas = 0;
let elsa = 0;
const audit = (category: string, action: string, extra: { targetType?: string; targetId?: string; detail?: unknown; actor?: string; at: string }) =>
db.insert(auditLog).values({ category, action, targetType: extra.targetType ?? null, targetId: extra.targetId ?? null, detail: extra.detail === undefined ? null : JSON.stringify(extra.detail), actorLabel: extra.actor ?? "Ada", createdAt: extra.at });
const ago = (min: number) => new Date(Date.now() - min * 60_000).toISOString().slice(0, 19).replace("T", " ");
before(async () => {
await db.insert(users).values([
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
const made = await db.insert(servers).values([{ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n" }, { name: "elsa", apiTokenHash: "e", apiTokenPrefix: "e" }]).returning({ id: servers.id });
nas = made[0].id;
elsa = made[1].id;
await audit("maintenance", "start", { targetType: "server", targetId: String(nas), detail: { name: "nas", minutes: 60, reason: "reboot" }, at: ago(10) });
await audit("server", "agent_updated", { targetType: "server", targetId: String(elsa), detail: { from: 4, to: 6 }, actor: "system", at: ago(20) });
await audit("dependency", "create", { targetType: "dependency", targetId: "1", detail: { from: "Gitea", to: "nas", fromKey: "integration:1", toKey: `server:${nas}` }, at: ago(30) });
await audit("settings", "update", { targetType: "settings", detail: { sections: ["display"], changes: { display: { pageSize: { from: 20, to: 50 } } } }, at: ago(40) });
await audit("session", "login", { at: ago(5) });
await audit("privacy", "export_own_data", { at: ago(6) });
await audit("inventory", "create", { targetType: "asset", targetId: "7", detail: { name: "Core switch", kind: "network" }, at: ago(60 * 24 * 3) });
await audit("server", "update", { targetType: "server", targetId: String(nas), detail: { name: "nas" }, at: ago(60 * 24 * 20) });
await db.insert(alertIncidents).values({ alertKey: "offline:offline:server:1", severity: "critical", peakSeverity: "critical", category: "offline", source: "Server", message: "nas hasn't reported for 3 h", targetType: "server", targetId: nas, targetName: "nas", openedAt: new Date(Date.now() - 50 * 60_000).toISOString(), resolvedAt: new Date(Date.now() - 15 * 60_000).toISOString(), lastSeenAt: new Date().toISOString() });
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/changes", changesRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/changes`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const get = async (query = "") => (await fetch(`${base}${query}`)).json();
const titles = (r: { entries: { title: string }[] }) => r.entries.map((e) => e.title);
test("everything of the last week, newest first, with the alerts raised and cleared in between; sign-ins and data exports are not changes", async () => {
const r = await get();
assert.deepEqual(titles(r), [
"Maintenance started on nas",
"Alert cleared: nas hasn't reported for 3 h",
"Agent updated to version 6",
"Gitea now needs nas",
"Settings changed: display",
"Alert raised: nas hasn't reported for 3 h",
"Hardware added: Core switch",
]);
assert.ok(!r.entries.some((e: { title: string }) => /login|export/i.test(e.title)));
const maint = r.entries[0];
assert.deepEqual([maint.kind, maint.actor, maint.link, maint.serverId], ["maintenance", "Ada", `/servers/${nas}`, nas]);
assert.equal(r.entries.find((e: { title: string }) => e.title.startsWith("Alert raised")).link.startsWith("/incidents/"), true);
assert.equal(r.kinds.length, 10);
});
test("by kind, by how far back, and by a word", async () => {
assert.deepEqual(titles(await get("?kind=maintenance")), ["Maintenance started on nas"]);
assert.deepEqual(titles(await get("?kind=alerts")).length, 2);
assert.equal((await get("?days=1")).entries.length, 6, "the hardware added three days ago is out");
assert.equal((await get("?days=30")).entries.length, 8, "and a month covers the old edit too");
assert.deepEqual(titles(await get("?q=gitea")), ["Gitea now needs nas"]);
assert.deepEqual(titles(await get("?q=version%20elsa")), ["Agent updated to version 6"], "every word must appear somewhere, the server's name included");
assert.deepEqual((await get("?kind=nonsense")).entries.length, 7, "an unknown kind is no filter");
});
test("by server: its own entries, its alerts, and what mentions it by key — not the other server's", async () => {
assert.deepEqual(titles(await get(`?serverId=${nas}`)), ["Maintenance started on nas", "Alert cleared: nas hasn't reported for 3 h", "Gitea now needs nas", "Alert raised: nas hasn't reported for 3 h"]);
assert.deepEqual(titles(await get(`?serverId=${elsa}`)), ["Agent updated to version 6"]);
assert.deepEqual((await get("?serverId=99999")).entries, []);
});
test("paged: a limit says there is more, and 'before' goes on from where it stopped", async () => {
const first = await get("?limit=3");
assert.equal(first.entries.length, 3);
assert.equal(first.more, true);
const next = await get(`?limit=3&before=${encodeURIComponent(first.entries[2].at)}`);
assert.deepEqual(titles(next), ["Gitea now needs nas", "Settings changed: display", "Alert raised: nas hasn't reported for 3 h"]);
assert.ok(next.entries.every((e: { at: string }) => e.at < first.entries[2].at));
assert.equal(next.more, true, "the hardware added three days ago is still to come");
const last = await get(`?limit=3&before=${encodeURIComponent(next.entries[2].at)}`);
assert.deepEqual(titles(last), ["Hardware added: Core switch"]);
assert.equal(last.more, false);
});
test("operators and administrators can read it; a viewer can't", async () => {
who = "viewer";
assert.equal((await fetch(base)).status, 403);
who = "operator";
assert.equal((await fetch(base)).status, 200);
});
test("the service on its own gives the same answer for any moment it is asked about", async () => {
const future = await getChangeLog({ now: new Date(Date.now() + 40 * 86_400_000), days: 1 });
assert.deepEqual(future.entries, [], "nothing in the day before a month from now");
});
});
+407
View File
@@ -0,0 +1,407 @@
/**
* Endpoint checks: what counts as an answer, the probes themselves (against servers this machine runs), what a run records, the problems they
* stand for, and what the routes, the alerts and the notifications do with them.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { createServer as createHttpServer, type Server } from "node:http";
import { createServer as createHttpsServer } from "node:https";
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { networkInterfaces, tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-endpoints-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, endpointChecks, endpointResults } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { statusAccepter, isStatusSpec, summariseResults, endpointConditions, probe, runCheck, runDueChecks } = await import("../src/services/endpointChecks.js");
const { runEndpointPass } = await import("../src/services/endpointMonitor.js");
const { endpointsRouter } = await import("../src/routes/endpoints.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { startOrExtendWindow } = await import("../src/services/maintenance.js");
const { updateSettings } = await import("../src/services/settingsStore.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
type Row = typeof endpointChecks.$inferSelect;
const NOW = new Date("2026-10-18T12:00:00Z");
const row = (extra: Partial<Row> = {}): Row => ({
id: 1, name: "Gitea", kind: "http", target: "https://git.lab/", port: null, expectStatus: null, keyword: null, insecure: false, intervalMinutes: 5, timeoutSeconds: 10, failuresBeforeAlert: 2, warnDays: 14,
serverId: null, enabled: true, lastCheckedAt: null, lastOk: null, lastMessage: null, lastResponseMs: null, certExpiresAt: null, consecutiveFailures: 0, lastChangeAt: null, createdAt: "2026-10-01 00:00:00", ...extra,
});
describe("what counts as an answer", () => {
test("status codes, ranges and the default of 200 to 399", () => {
const usual = statusAccepter(null);
assert.deepEqual([199, 200, 301, 399, 400, 500].map(usual), [false, true, true, true, false, false]);
const custom = statusAccepter("200-204, 401");
assert.deepEqual([200, 204, 205, 401, 403].map(custom), [true, true, false, true, false]);
assert.equal(statusAccepter("rubbish")(200), true, "unreadable falls back to the usual");
assert.equal(isStatusSpec("200, 301-302"), true);
assert.equal(isStatusSpec("2xx"), false);
});
test("uptime, the average answer time and the recent run come from the stored results", () => {
const at = (min: number) => new Date(NOW.getTime() - min * 60_000).toISOString();
const stats = summariseResults(
[
{ at: at(10), ok: true, ms: 100 },
{ at: at(20), ok: true, ms: 300 },
{ at: at(30), ok: false, ms: null },
{ at: at(60 * 30), ok: false, ms: null }, // yesterday and a bit
{ at: at(60 * 24 * 6), ok: true, ms: 50 },
],
NOW,
);
assert.equal(stats.uptime24h, 66.7, "2 of 3 in the last day");
assert.equal(stats.uptime7d, 60, "3 of 5 in the last week");
assert.equal(stats.avgMs24h, 200, "only answers that came count");
assert.equal(stats.recent[0].at, at(60 * 24 * 6), "oldest first");
assert.deepEqual(summariseResults([], NOW), { uptime24h: null, uptime7d: null, avgMs24h: null, recent: [] });
});
});
describe("what is a problem", () => {
test("a check has to fail several times in a row; a blip isn't", () => {
assert.deepEqual(endpointConditions([row({ lastOk: false, consecutiveFailures: 1, lastMessage: "answered 500" })], NOW), []);
const [c] = endpointConditions([row({ lastOk: false, consecutiveFailures: 2, lastMessage: "answered 500" })], NOW);
assert.equal(c.key, "endpoint:1");
assert.equal(c.category, "monitoring");
assert.equal(c.severity, "critical");
assert.match(c.message, /Gitea isn't answering — answered 500/);
assert.deepEqual(endpointConditions([row({ lastOk: false, consecutiveFailures: 9, enabled: false })], NOW), [], "a check that is switched off says nothing");
assert.deepEqual(endpointConditions([row({ lastOk: true, consecutiveFailures: 0 })], NOW), []);
});
test("a certificate is a problem from its warning time on, critical in the last three days, and the key carries the server it is linked to", () => {
const days = (n: number) => new Date(NOW.getTime() + n * 86_400_000).toISOString();
assert.deepEqual(endpointConditions([row({ certExpiresAt: days(20) })], NOW), []);
const [warn] = endpointConditions([row({ certExpiresAt: days(10), serverId: 4 })], NOW);
assert.equal(warn.key, "endpoint-cert:1:server:4");
assert.deepEqual([warn.category, warn.severity], ["expiry", "warning"]);
assert.match(warn.message, /expires in 10 days/);
assert.equal(endpointConditions([row({ certExpiresAt: days(2) })], NOW)[0].severity, "critical");
assert.match(endpointConditions([row({ certExpiresAt: days(-3) })], NOW)[0].message, /expired 3 days ago/);
});
test("a certificate check whose certificate has expired says so once, not also that it isn't answering", () => {
const expired = new Date(NOW.getTime() - 86_400_000).toISOString();
const list = endpointConditions([row({ kind: "tls", target: "git.lab", port: 443, lastOk: false, consecutiveFailures: 5, lastMessage: "The certificate expired", certExpiresAt: expired })], NOW);
assert.deepEqual(list.map((c) => c.category), ["expiry"]);
});
});
/** An address of this machine that isn't loopback (the checks refuse that), if it has one. */
function lanAddress(): string | null {
for (const list of Object.values(networkInterfaces())) {
for (const i of list ?? []) if (i.family === "IPv4" && !i.internal && !i.address.startsWith("169.254.")) return i.address;
}
return null;
}
const LAN = lanAddress();
describe("asking", { skip: LAN === null && "this machine has no address other than loopback" }, () => {
let web: Server;
let port = 0;
let sleeping: Server;
let sleepingPort = 0;
const hits: string[] = [];
before(async () => {
web = createHttpServer((req, res) => {
hits.push(req.url ?? "");
if (req.url === "/ok") return void res.end("<h1>Welcome to the lab</h1>");
if (req.url === "/broken") {
res.statusCode = 500;
return void res.end("oops");
}
if (req.url === "/moved") {
res.statusCode = 301;
res.setHeader("location", "/ok");
return void res.end();
}
res.statusCode = 404;
res.end("no");
});
await new Promise<void>((r) => web.listen(0, "0.0.0.0", r));
port = (web.address() as AddressInfo).port;
// Accepts the connection and never answers.
sleeping = createHttpServer(() => {});
await new Promise<void>((r) => sleeping.listen(0, "0.0.0.0", r));
sleepingPort = (sleeping.address() as AddressInfo).port;
});
after(() => {
web.close();
sleeping.close();
});
const http = (path: string, extra: Partial<Row> = {}) => row({ kind: "http", target: `http://${LAN}:${port}${path}`, timeoutSeconds: 3, ...extra });
test("a web address that answers is fine, with how long it took", async () => {
const r = await probe(http("/ok"));
assert.equal(r.ok, true);
assert.match(r.message, /answered 200 in \d+ ms/);
assert.ok(r.ms !== null && r.ms >= 0);
});
test("a wrong status fails, unless that is what was expected; a redirect is an answer", async () => {
assert.match((await probe(http("/broken"))).message, /answered 500/);
assert.equal((await probe(http("/broken", { expectStatus: "500" }))).ok, true);
assert.equal((await probe(http("/missing"))).ok, false);
assert.equal((await probe(http("/moved"))).ok, true, "301 is within 200 to 399");
});
test("a word the page must contain: found after following a redirect, or missing", async () => {
assert.equal((await probe(http("/ok", { keyword: "Welcome" }))).ok, true);
const missing = await probe(http("/ok", { keyword: "Goodbye" }));
assert.equal(missing.ok, false);
assert.match(missing.message, /doesn't contain “Goodbye”/);
assert.equal((await probe(http("/moved", { keyword: "Welcome" }))).ok, true, "with a word to find, the redirect is followed to the page");
});
test("no answer in time, and a refused connection, say so", async () => {
const slow = await probe(row({ kind: "http", target: `http://${LAN}:${sleepingPort}/`, timeoutSeconds: 1 }));
assert.equal(slow.ok, false);
assert.match(slow.message, /no answer within 1 s/);
const refused = await probe(row({ kind: "tcp", target: LAN!, port: 1, timeoutSeconds: 2 }));
assert.equal(refused.ok, false);
const open = await probe(row({ kind: "tcp", target: LAN!, port, timeoutSeconds: 2 }));
assert.equal(open.ok, true);
assert.match(open.message, new RegExp(`port ${port} answered in \\d+ ms`));
});
test("it will not be pointed at this machine itself or at the cloud's metadata address", async () => {
for (const target of ["http://127.0.0.1:1/", "http://169.254.169.254/latest/meta-data/"]) {
const r = await probe(row({ kind: "http", target }));
assert.equal(r.ok, false);
assert.match(r.message, /can't be used|isn't allowed/);
}
assert.equal((await probe(row({ kind: "tcp", target: "127.0.0.1", port: 22 }))).ok, false);
assert.equal((await probe(row({ kind: "http", target: "ftp://example.com/" }))).ok, false);
});
test("a certificate that can't be trusted fails unless it is accepted, and either way its expiry is read", async () => {
const work = mkdtempSync(join(tmpdir(), "hm-cert-"));
const made = spawnSync("openssl", ["req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "30", "-subj", "/CN=lab.example", "-keyout", join(work, "k.pem"), "-out", join(work, "c.pem")], { encoding: "utf8" });
if (made.status !== 0) {
rmSync(work, { recursive: true, force: true });
return; // no openssl here: the rest is covered by what the pure parts say
}
const secure = createHttpsServer({ key: readFileSync(join(work, "k.pem")), cert: readFileSync(join(work, "c.pem")) }, (_req, res) => res.end("secure"));
await new Promise<void>((r) => secure.listen(0, "0.0.0.0", r));
const sp = (secure.address() as AddressInfo).port;
try {
const strict = await probe(row({ kind: "http", target: `https://${LAN}:${sp}/`, timeoutSeconds: 5 }));
assert.equal(strict.ok, false);
assert.match(strict.message, /certificate can't be trusted: it is self-signed/);
const lenient = await probe(row({ kind: "http", target: `https://${LAN}:${sp}/`, insecure: true, timeoutSeconds: 5 }));
assert.equal(lenient.ok, true);
assert.ok(lenient.certExpiresAt && Date.parse(lenient.certExpiresAt) > Date.now() + 25 * 86_400_000, "about 30 days left");
const tlsStrict = await probe(row({ kind: "tls", target: LAN!, port: sp, timeoutSeconds: 5 }));
assert.equal(tlsStrict.ok, false);
assert.match(tlsStrict.message, /can't be trusted/);
const tlsLenient = await probe(row({ kind: "tls", target: LAN!, port: sp, insecure: true, timeoutSeconds: 5 }));
assert.equal(tlsLenient.ok, true);
assert.match(tlsLenient.message, /good until \d{4}-\d{2}-\d{2}/);
const plain = await probe(row({ kind: "tls", target: LAN!, port, timeoutSeconds: 5 }));
assert.equal(plain.ok, false);
assert.match(plain.message, /isn't speaking TLS/);
} finally {
secure.close();
rmSync(work, { recursive: true, force: true });
}
});
});
describe("a run, a pass, the routes, the alerts and the notifications", () => {
let app: Server;
let base = "";
let who = "admin";
let serverId = 0;
const messages: string[] = [];
let hook: Server;
let answer: { ok: boolean; ms: number | null; message: string; certExpiresAt?: string | null } = { ok: true, ms: 12, message: "answered 200 in 12 ms" };
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
serverId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
hook = createHttpServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c) => chunks.push(c));
req.on("end", () => {
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
res.end("{}");
});
});
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/endpoints", endpointsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/endpoints`;
});
after(() => {
app.close();
hook.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
const fake = async () => answer;
let id = 0;
test("an administrator adds a check; a wrong one is refused with a reason; nobody else can", async () => {
const res = await send("POST", "/", { name: "Gitea", kind: "http", target: "https://git.lab/", serverId });
assert.equal(res.status, 201);
id = (await res.json()).check.id;
for (const bad of [
{ name: "x", kind: "http", target: "git.lab" },
{ name: "x", kind: "http", target: "https://a/", expectStatus: "2xx" },
{ name: "x", kind: "tcp", target: "host" },
{ name: "x", kind: "tcp", target: "http://host", port: 80 },
{ name: "x", kind: "tls", target: "host", port: 70000 },
{ name: "x", kind: "ping", target: "host" },
{ name: "x", kind: "http", target: "https://a/", serverId: 99999 },
]) assert.equal((await send("POST", "/", bad)).status, 400, JSON.stringify(bad));
who = "operator";
assert.equal((await send("POST", "/", { name: "y", kind: "tcp", target: "h", port: 22 })).status, 403);
who = "viewer";
assert.equal((await send("GET", "/")).status, 200, "everyone can look");
who = "admin";
});
test("a run records the answer on the check and as a point in its history", async () => {
const [c] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id));
answer = { ok: true, ms: 40, message: "answered 200 in 40 ms", certExpiresAt: new Date(NOW.getTime() + 3 * 86_400_000).toISOString() };
await runCheck(c, new Date("2026-10-18T12:00:00Z"), fake);
const [after1] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id));
assert.deepEqual([after1.lastOk, after1.lastResponseMs, after1.consecutiveFailures, after1.lastChangeAt], [true, 40, 0, "2026-10-18T12:00:00.000Z"]);
assert.ok(after1.certExpiresAt);
assert.equal((await db.select().from(endpointResults).where(eq(endpointResults.checkId, id))).length, 1);
answer = { ok: false, ms: null, message: "answered 500" };
await runCheck(after1, new Date("2026-10-18T12:05:00Z"), fake);
const [failed] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id));
assert.deepEqual([failed.lastOk, failed.consecutiveFailures, failed.lastChangeAt, failed.lastMessage], [false, 1, "2026-10-18T12:05:00.000Z", "answered 500"]);
await runCheck(failed, new Date("2026-10-18T12:10:00Z"), fake);
const [again] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id));
assert.deepEqual([again.consecutiveFailures, again.lastChangeAt], [2, "2026-10-18T12:05:00.000Z"], "the time of the change is when it began failing, not each failure");
});
test("only the checks that are due are asked", async () => {
await db.update(endpointChecks).set({ lastCheckedAt: "2026-10-18T12:10:00.000Z" }).where(eq(endpointChecks.id, id));
let asked = 0;
const counting = async () => {
asked++;
return answer;
};
assert.equal(await runDueChecks(new Date("2026-10-18T12:12:00Z"), counting), 0, "two minutes after a five-minute check");
assert.equal(await runDueChecks(new Date("2026-10-18T12:15:30Z"), counting), 1);
assert.equal(asked, 1);
await db.update(endpointChecks).set({ enabled: false }).where(eq(endpointChecks.id, id));
assert.equal(await runDueChecks(new Date("2026-10-18T13:00:00Z"), counting), 0, "a check that is switched off is never asked");
await db.update(endpointChecks).set({ enabled: true }).where(eq(endpointChecks.id, id));
});
test("a check that keeps failing is an alert, silenced by maintenance on its server, and a certificate running out is another", async () => {
await db.update(endpointChecks).set({ lastOk: false, consecutiveFailures: 3, lastMessage: "answered 500", certExpiresAt: new Date(Date.now() + 2 * 86_400_000).toISOString() }).where(eq(endpointChecks.id, id));
invalidateAlertsCache();
let alerts = (await getAlerts(true)).alerts.filter((a) => a.source === "Endpoint");
const down = alerts.find((a) => a.category === "monitoring")!;
const cert = alerts.find((a) => a.category === "expiry")!;
assert.match(down.message, /Gitea isn't answering — answered 500/);
assert.equal(down.severity, "critical");
assert.equal(down.link, "/endpoints");
assert.deepEqual(down.target && [down.target.type, down.target.id], ["server", serverId]);
assert.match(cert.message, /expires in/);
assert.equal(cert.severity, "critical", "two days left");
assert.equal(down.silenced, false);
await startOrExtendWindow("server", serverId, 60, "work", "Ada");
invalidateAlertsCache();
alerts = (await getAlerts(true)).alerts.filter((a) => a.source === "Endpoint");
assert.ok(alerts.length === 2 && alerts.every((a) => a.silenced), "both are under the server's maintenance");
});
test("the pass tells you once when a check starts failing, and once when it is fine again", async () => {
const { listActiveWindows } = await import("../src/services/maintenance.js");
const { maintenanceWindows } = await import("../src/db/schema.js");
for (const w of await listActiveWindows()) await db.delete(maintenanceWindows).where(eq(maintenanceWindows.id, w.id));
await db.update(endpointChecks).set({ lastOk: true, consecutiveFailures: 0, lastCheckedAt: null, certExpiresAt: null }).where(eq(endpointChecks.id, id));
messages.length = 0;
answer = { ok: false, ms: null, message: "answered 503" };
const t = (min: number) => new Date(Date.now() + min * 60_000);
let r = await runEndpointPass(t(0), fake);
assert.equal(r.announced, 0, "the first failure is a blip");
r = await runEndpointPass(t(6), fake);
assert.equal(r.announced, 1, "the second is a problem");
assert.match(messages.join("\n"), /Gitea isn't answering — answered 503/);
const sent = messages.length;
await runEndpointPass(t(12), fake);
assert.equal(messages.length, sent, "and it isn't said again");
answer = { ok: true, ms: 9, message: "answered 200 in 9 ms" };
r = await runEndpointPass(t(18), fake);
assert.equal(r.recovered, 1);
assert.match(messages[messages.length - 1], /Gitea isn't answering/, "the recovery names what cleared");
});
test("check now is for operators and answers at once; the list carries the figures; metrics have them", async () => {
who = "viewer";
assert.equal((await send("POST", `/${id}/check`)).status, 403);
who = "operator";
// the real probe, aimed at a name nothing answers to (and refused if it were loopback): it says what happened
await db.update(endpointChecks).set({ target: "http://127.0.0.1:1/" }).where(eq(endpointChecks.id, id));
const res = await send("POST", `/${id}/check`);
assert.equal(res.status, 200);
assert.equal((await res.json()).result.ok, false);
who = "viewer";
const list = await (await send("GET", "/")).json();
const c = list.checks.find((x: { id: number }) => x.id === id);
assert.equal(c.server.name, "nas");
assert.ok(c.stats.recent.length >= 1);
assert.equal(typeof c.stats.uptime24h, "number");
assert.match(await buildMetrics(), /homelab_endpoint_up\{name="Gitea",kind="http"\} 0/);
});
test("changing what is asked forgets what was found; removing it takes its history", async () => {
who = "admin";
const res = await send("PUT", `/${id}`, { name: "Gitea", kind: "http", target: "https://git2.lab/", serverId });
assert.equal(res.status, 200);
const [c] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id));
assert.deepEqual([c.lastOk, c.consecutiveFailures, c.lastMessage], [null, 0, null]);
assert.equal((await send("DELETE", `/${id}`)).status, 204);
assert.equal((await db.select().from(endpointResults).where(eq(endpointResults.checkId, id))).length, 0);
assert.equal((await send("DELETE", `/${id}`)).status, 404);
});
});
+210
View File
@@ -0,0 +1,210 @@
/**
* Restore tests: what the newest test of something amounts to, when to speak up, and the routes that record them.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-restore-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, restoreTests, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { restoreInfo, restoreAlerts, customKey, guestKey } = await import("../src/services/restoreTests.js");
const { setCoverageLoaderForTests, buildCoverage } = await import("../src/services/backupCoverage.js");
const { backupsRouter } = await import("../src/routes/backups.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { updateSettings } = await import("../src/services/settingsStore.js");
const { buildMetrics } = await import("../src/services/metrics.js");
await runMigrations();
const NOW = new Date("2026-10-18T12:00:00Z");
const daysAgo = (n: number) => new Date(NOW.getTime() - n * 86_400_000).toISOString().slice(0, 10);
const test1 = (result: string, ago: number, extra: Record<string, unknown> = {}) => ({ result, testedAt: daysAgo(ago), note: null as string | null, testedBy: "Ada", ...extra });
describe("what the newest test amounts to", () => {
test("how it went; none at all is never; an old pass is stale once there is a limit; a failure or 'not needed' doesn't age", () => {
assert.equal(restoreInfo(undefined, NOW, 90).status, "never");
assert.deepEqual(restoreInfo(test1("ok", 10), NOW, 90), { status: "ok", testedAt: daysAgo(10), ageDays: 10, note: null, testedBy: "Ada" });
assert.equal(restoreInfo(test1("ok", 100), NOW, 90).status, "stale");
assert.equal(restoreInfo(test1("ok", 100), NOW, 0).status, "ok", "no limit, no staleness");
assert.equal(restoreInfo(test1("partial", 100), NOW, 90).status, "stale");
assert.equal(restoreInfo(test1("failed", 400), NOW, 90).status, "failed");
assert.equal(restoreInfo(test1("not_needed", 400), NOW, 90).status, "not_needed");
});
test("something that isn't a guest is told apart by its name", () => {
assert.equal(customKey("Photos share"), "custom:photos-share");
assert.equal(customKey(" Gitea — database! "), "custom:gitea-database");
assert.equal(customKey("Åkerblom's NAS"), "custom:akerblom-s-nas");
assert.equal(customKey("!!!"), null);
assert.equal(guestKey(2, 105), "guest:2:105");
});
});
const guest = (vmid: number, name: string) => ({ vmid, name, node: "pve1", type: "qemu" as const, status: "running", cpu: null, maxmem: null, mem: null, uptime: null });
const fixture = () => {
const loaded = {
proxmox: [{ integrationId: 1, integrationName: "Felicia", guests: [guest(100, "elsa"), guest(101, "anna"), guest(102, "zoe")], jobs: [{ id: "j", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null }], tasks: [] }],
pbs: [{ integrationName: "Backup server", groups: [{ backupType: "vm", backupId: "100", backupTime: Math.floor(Date.now() / 1000) - 3600, verification: "ok" as const }, { backupType: "vm", backupId: "101", backupTime: Math.floor(Date.now() / 1000) - 7200, verification: "ok" as const }] }],
sources: [{ kind: "proxmox" as const, name: "Felicia", ok: true }],
};
return loaded;
};
describe("when to speak up", () => {
const report = () => ({ guests: buildCoverage({ ...fixture(), servers: [], now: NOW }), counts: {} as never, sources: [], staleHours: 48, generatedAt: NOW.toISOString() });
const row = (key: string, label: string, result: string, ago: number) => [key, { id: 1, subjectKey: key, label, testedAt: daysAgo(ago), result, note: null, testedBy: null, createdAt: "" }] as const;
test("with no limit only a failure speaks; with one, a guest that has backups and no recent test does too", () => {
const g = report();
assert.deepEqual(restoreAlerts(g, new Map(), 0, NOW), [], "no limit, nothing tested: quiet");
const limited = restoreAlerts(g, new Map(), 90, NOW);
assert.deepEqual(limited.map((a) => [a.guest?.name, a.severity]), [["elsa", "warning"], ["anna", "warning"]], "zoe has no backups to restore from, so nothing to say");
assert.match(limited[0].message, /elsa \(#100\) \[Felicia\]: a restore has never been tested/);
const failed = restoreAlerts(g, new Map([row("guest:1:100", "elsa", "failed", 3)]), 0, NOW);
assert.equal(failed.length, 1);
assert.equal(failed[0].severity, "critical");
assert.match(failed[0].message, /the last restore test \(.*\) failed/);
});
test("a recent pass, 'not needed' and an old pass under the limit are all fine; an old one isn't", () => {
const g = report();
const tests = new Map([row("guest:1:100", "elsa", "ok", 10), row("guest:1:101", "anna", "not_needed", 500)]);
assert.deepEqual(restoreAlerts(g, tests, 90, NOW), []);
const old = restoreAlerts(g, new Map([row("guest:1:100", "elsa", "ok", 120), row("guest:1:101", "anna", "ok", 1)]), 90, NOW);
assert.equal(old.length, 1);
assert.match(old[0].message, /last restore test was 120 days ago \(the limit is 90 days\)/);
});
test("something that isn't a guest is told of when it failed, or when it is stale", () => {
const g = report();
const tests = new Map([row("custom:photos", "Photos share", "ok", 200), row("custom:db", "Gitea database", "failed", 5), row("custom:fine", "Fine", "ok", 3), row("guest:9:9", "gone guest", "ok", 999)]);
const out = restoreAlerts(g, tests, 90, NOW);
assert.deepEqual(out.map((a) => a.message.split(":")[0]).sort(), ["Gitea database", "Photos share", "anna (#101) [Felicia]", "elsa (#100) [Felicia]"].sort());
assert.equal(restoreAlerts(g, tests, 0, NOW).filter((a) => !a.guest).length, 1, "with no limit only the failure");
});
});
describe("through the routes, the page, the alerts and the figures", () => {
let app: Server;
let base = "";
let who = "operator";
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
setCoverageLoaderForTests(async () => fixture());
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/backups", backupsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/backups`;
});
after(() => {
setCoverageLoaderForTests(null);
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
const today = new Date().toISOString().slice(0, 10);
test("an operator records a test; a viewer can't; a wrong one is refused", async () => {
assert.equal((await send("POST", "/restore-tests", { key: "guest:1:100", label: "elsa", result: "ok", note: "booted and the data was there" })).status, 201);
who = "viewer";
assert.equal((await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "ok" })).status, 403);
who = "operator";
for (const bad of [
{ key: "guest:1:101", label: "anna", result: "excellent" },
{ key: "guest:one:two", label: "anna", result: "ok" },
{ key: "../etc", label: "anna", result: "ok" },
{ key: "guest:1:101", label: "anna", result: "ok", testedAt: "2999-01-01" },
{ key: "guest:1:101", label: "anna", result: "ok", testedAt: "2026-02-30" },
{ label: "!!!", result: "ok" },
{ key: "guest:1:101", label: "", result: "ok" },
]) assert.equal((await send("POST", "/restore-tests", bad)).status, 400, JSON.stringify(bad));
assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "restore_test" && e.action === "record").length, 1);
});
test("the Backups page's data says how each guest's restore went, or that nobody has tried", async () => {
const cov = await (await send("GET", "/coverage")).json();
const by = Object.fromEntries(cov.guests.map((g: { name: string }) => [g.name, g]));
assert.equal(by.elsa.restoreKey, "guest:1:100");
assert.deepEqual([by.elsa.restoreTest.status, by.elsa.restoreTest.testedAt, by.elsa.restoreTest.note], ["ok", today, "booted and the data was there"]);
assert.equal(by.anna.restoreTest.status, "never");
assert.equal(cov.restoreTestDays, 0);
});
test("something that isn't a guest gets a key from its name, and is listed with how it stands", async () => {
const res = await send("POST", "/restore-tests", { label: "Photos share", result: "ok", testedAt: daysAgo(40) });
assert.equal(res.status, 201);
assert.equal((await res.json()).test.subjectKey, "custom:photos-share");
const list = await (await send("GET", "/restore-tests")).json();
assert.deepEqual(list.custom.map((c: { key: string; status: string }) => [c.key, c.status]), [["custom:photos-share", "ok"]]);
assert.equal(list.tests.length, 2);
});
test("a failed test is an alert straight away; with a limit, never-tested and old ones are too; marking 'not needed' ends it", async () => {
await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "failed", note: "the disk wouldn't attach" });
invalidateAlertsCache();
let alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:"));
assert.deepEqual(alerts.map((a) => [a.id, a.severity]), [["backup:restore-test:guest:1:101", "critical"]]);
assert.match(alerts[0].message, /anna \(#101\).*failed — the disk wouldn't attach/);
assert.equal(alerts[0].link, "/backups");
await updateSettings({ healthChecks: { restoreTestDays: 30 } });
invalidateAlertsCache();
alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:"));
assert.deepEqual(alerts.map((a) => a.id).sort(), ["backup:restore-test:custom:photos-share", "backup:restore-test:guest:1:101"], "elsa is fine; the photos share was last tested 40 days ago");
await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "not_needed", note: "a scratch machine" });
invalidateAlertsCache();
alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:"));
assert.deepEqual(alerts.map((a) => a.id), ["backup:restore-test:custom:photos-share"]);
await updateSettings({ healthChecks: { restoreTestDays: 0 } });
});
test("metrics say how old each test is and whether it passed", async () => {
const text = await buildMetrics();
assert.match(text, /homelab_restore_test_age_days\{subject="elsa"\} 0/);
assert.match(text, /homelab_restore_test_age_days\{subject="Photos share"\} 40/);
assert.match(text, /homelab_restore_test_passed\{subject="elsa"\} 1/);
assert.doesNotMatch(text, /subject="anna"/, "'not needed' isn't measured");
});
test("only an administrator takes an entry out, and that is recorded", async () => {
const [first] = await db.select().from(restoreTests);
assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 403);
who = "admin";
assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 204);
assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 404);
assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "restore_test" && e.action === "delete").length, 1);
});
});