Files
Homelab-manager/server/test/metricsAndWebhooks.test.ts
T
bobbanandClaude Sonnet 5.5 c17e9aa140 Add runbooks, backup coverage, inventory, roles from groups, Prometheus metrics and event webhooks
- Runbooks: notes per server/integration, shown next to matching alerts
- Backup coverage page across Proxmox and PBS, with a digest section
- Hardware inventory with warranty, power and running cost
- Authentik group to role mapping with preview and safeguards
- /metrics (bearer token) and signed outgoing event webhooks
- Docs: README, DATABASE, NOTIFICATIONS, ROLES and new API.md

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-17 20:41:00 +02:00

323 lines
17 KiB
TypeScript

/**
* The Prometheus endpoint and its tokens, and the signed event webhooks.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import http from "node:http";
import { createHmac } from "node:crypto";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-metrics-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, secrets, maintenanceWindows, metricsTokens, eventWebhooks, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { metricsRouter, metricsTokensRouter } = await import("../src/routes/metrics.js");
const { eventWebhooksRouter } = await import("../src/routes/eventWebhooks.js");
const { escapeLabel } = await import("../src/services/metrics.js");
const { deliver, emitEvent, signBody } = await import("../src/services/eventWebhooks.js");
const { checkMaintenanceEvents, resetMaintenanceEventsForTests } = await import("../src/services/maintenanceEvents.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
describe("escaping", () => {
test("a label value keeps its quotes, backslashes and new lines from breaking the line", () => {
assert.equal(escapeLabel('say "hi"'), 'say \\"hi\\"');
assert.equal(escapeLabel("C:\\temp"), "C:\\\\temp");
assert.equal(escapeLabel("a\nb"), "a\\nb");
});
});
describe("metrics and webhooks", () => {
let app: http.Server;
let base = "";
let who = "admin";
let receiver: http.Server;
const received: { headers: http.IncomingHttpHeaders; body: string }[] = [];
let receiverStatus: number[] = []; // statuses to answer with, in turn; 200 once it runs out
let receiverUrl = "";
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olga", role: "operator" },
]);
const now = new Date();
await db.insert(servers).values([
{
name: 'elsa "main"',
apiTokenHash: "a",
apiTokenPrefix: "a",
lastSeenAt: now.toISOString(),
cpuLoadPercent: 12.5,
memTotalBytes: 1000,
memUsedBytes: 250,
disks: JSON.stringify([{ mount: "C:\\", sizeBytes: 100, usedBytes: 40 }]),
pendingUpdates: JSON.stringify({ manager: "apt", count: 7, security: 2, rebootRequired: true, reportedAt: now.toISOString() }),
agentVersion: 1,
},
{ name: "anna", apiTokenHash: "b", apiTokenPrefix: "b", lastSeenAt: new Date(now.getTime() - 5 * 3600_000).toISOString() },
{ name: "never", apiTokenHash: "c", apiTokenPrefix: "c" },
]);
await db.insert(secrets).values({ name: "VPN key", type: "api_token", expiryDate: new Date(now.getTime() + 9 * 86_400_000).toISOString().slice(0, 10), warnDays: 30 });
receiver = await new Promise<http.Server>((resolve) => {
const s = http.createServer((req, res) => {
let raw = "";
req.on("data", (c) => (raw += c));
req.on("end", () => {
received.push({ headers: req.headers, body: raw });
res.statusCode = receiverStatus.shift() ?? 200;
res.end("ok");
});
});
s.listen(0, "127.0.0.1", () => resolve(s));
});
receiverUrl = `http://127.0.0.1:${(receiver.address() as AddressInfo).port}/hook`;
const a = express();
a.use("/metrics", metricsRouter); // no session: it is the bearer token that counts
const api = express();
api.use(express.json());
api.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
api.use("/api/metrics-tokens", metricsTokensRouter);
api.use("/api/event-webhooks", eventWebhooksRouter);
a.use(api);
app = await new Promise<http.Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
});
after(() => {
app.close();
receiver.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const send = (method: string, path: string, body?: unknown) =>
fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
let secret = "";
let tokenId = 0;
test("an administrator makes a token; the secret is shown once and only its hash is kept", async () => {
const res = await send("POST", "/api/metrics-tokens", { label: "Prometheus" });
assert.equal(res.status, 201);
const body = await res.json();
secret = body.secret;
tokenId = body.token.id;
assert.match(secret, /^hlm_m_/);
const list = await (await send("GET", "/api/metrics-tokens")).json();
assert.ok(!JSON.stringify(list).includes(secret) && !JSON.stringify(list).includes("tokenHash"));
const [stored] = await db.select().from(metricsTokens);
assert.notEqual(stored.tokenHash, secret);
assert.equal(stored.tokenHash.length, 64);
who = "operator";
assert.equal((await send("POST", "/api/metrics-tokens", { label: "x" })).status, 403);
assert.equal((await send("GET", "/api/metrics-tokens")).status, 403);
who = "admin";
assert.equal((await send("POST", "/api/metrics-tokens", { label: "" })).status, 400);
});
test("/metrics wants the token in a header: nothing, a wrong one, or one in the address all get the same refusal", async () => {
const none = await fetch(`${base}/metrics`);
assert.equal(none.status, 401);
assert.match(none.headers.get("www-authenticate") ?? "", /Bearer/);
const refusal = await none.text();
assert.equal((await fetch(`${base}/metrics`, { headers: { authorization: "Bearer hlm_m_wrongwrongwrongwrongwrong" } })).status, 401);
assert.equal((await fetch(`${base}/metrics?token=${secret}`)).status, 401, "a token in the address is not accepted");
assert.equal(await (await fetch(`${base}/metrics`, { headers: { authorization: "Bearer nope" } })).text(), refusal);
});
test("with the token it answers in Prometheus's format, with the servers, alerts and the rest", async () => {
const res = await fetch(`${base}/metrics`, { headers: { authorization: `Bearer ${secret}` } });
assert.equal(res.status, 200);
assert.match(res.headers.get("content-type") ?? "", /^text\/plain; version=0\.0\.4/);
assert.equal(res.headers.get("cache-control"), "no-store");
const text = await res.text();
assert.match(text, /# TYPE homelab_server_up gauge/);
assert.match(text, /homelab_server_up\{server="elsa \\"main\\""\} 1/, "the name's quotes are escaped");
assert.match(text, /homelab_server_up\{server="anna"\} 0/, "five hours silent is past the offline threshold");
assert.ok(!text.includes('server="never"'), "a server that never reported has nothing to say");
assert.match(text, /homelab_server_cpu_load_percent\{server="elsa \\"main\\""\} 12\.5/);
assert.match(text, /homelab_server_memory_used_percent\{server="elsa \\"main\\""\} 25/);
assert.match(text, /homelab_server_disk_used_percent\{server="elsa \\"main\\"",mount="C:\\\\"\} 40/, "a backslash in a mount is escaped");
assert.match(text, /homelab_server_pending_updates\{server="elsa \\"main\\""\} 7/);
assert.match(text, /homelab_server_pending_security_updates\{server="elsa \\"main\\""\} 2/);
assert.match(text, /homelab_server_reboot_required\{server="elsa \\"main\\""\} 1/);
assert.match(text, /homelab_server_agent_outdated\{server="elsa \\"main\\""\} 1/, "version 1 is behind");
assert.match(text, /homelab_alerts\{severity="critical"\} \d+/);
assert.match(text, /homelab_secret_expiry_days\{name="VPN key"\} 9/);
assert.match(text, /homelab_maintenance_active_windows 0/);
assert.match(text, /homelab_scrape_duration_seconds /);
});
test("every line is valid exposition format, and each metric is described once", async () => {
const text = await (await fetch(`${base}/metrics`, { headers: { authorization: `Bearer ${secret}` } })).text();
const helps: string[] = [];
for (const line of text.trim().split("\n")) {
if (line.startsWith("# HELP ")) {
helps.push(line.split(" ")[2]);
continue;
}
if (line.startsWith("# TYPE ")) continue;
assert.match(line, /^[a-zA-Z_:][a-zA-Z0-9_:]*(\{([a-zA-Z_][a-zA-Z0-9_]*="([^"\\]|\\.)*",?)*\})? -?\d+(\.\d+)?$/, line);
}
assert.equal(new Set(helps).size, helps.length, "no metric is declared twice");
});
test("use is noted, and a removed token stops working at once", async () => {
const [row] = await db.select().from(metricsTokens).where(eq(metricsTokens.id, tokenId));
assert.ok(row.lastUsedAt);
assert.equal((await send("DELETE", `/api/metrics-tokens/${tokenId}`)).status, 204);
assert.equal((await fetch(`${base}/metrics`, { headers: { authorization: `Bearer ${secret}` } })).status, 401);
assert.equal((await send("DELETE", `/api/metrics-tokens/${tokenId}`)).status, 404);
const entries = (await db.select().from(auditLog)).filter((e) => e.category === "metrics_token");
assert.deepEqual(entries.map((e) => e.action), ["create", "delete"]);
assert.ok(entries.every((e) => !(e.detail ?? "").includes(secret)));
});
// ── event webhooks ──
let hookId = 0;
const HOOK_SECRET = "s3cret-for-signing";
const waitFor = async (n: number, ms = 3000) => {
for (let t = 0; t < ms && received.length < n; t += 25) await new Promise((r) => setTimeout(r, 25));
};
test("a webhook is made with a secret that is never shown again, and kept encrypted", async () => {
const res = await send("POST", "/api/event-webhooks", { label: "Home Assistant", url: receiverUrl, secret: HOOK_SECRET, events: ["alert.opened", "maintenance.started"] });
assert.equal(res.status, 201);
const hook = (await res.json()).webhook;
hookId = hook.id;
assert.equal(hook.hasSecret, true);
assert.deepEqual(hook.events, ["alert.opened", "maintenance.started"]);
assert.ok(!JSON.stringify(await (await send("GET", "/api/event-webhooks")).json()).includes(HOOK_SECRET));
const [stored] = await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, hookId));
assert.match(stored.secret ?? "", /^enc:v1:/);
const audit = (await db.select().from(auditLog)).find((e) => e.category === "webhook" && e.action === "create");
assert.ok(!(audit?.detail ?? "").includes(receiverUrl) && !(audit?.detail ?? "").includes(HOOK_SECRET), "neither the address nor the secret goes in the audit log");
});
test("a bad webhook is refused: not http, a password in the address, no events, only admins", async () => {
for (const bad of [
{ label: "x", url: "ftp://x.example/hook", events: ["alert.opened"] },
{ label: "x", url: "https://user:pass@x.example/hook", events: ["alert.opened"] },
{ label: "x", url: "not a url", events: ["alert.opened"] },
{ label: "x", url: receiverUrl, events: [] },
{ label: "x", url: receiverUrl, events: ["alert.exploded"] },
{ label: "", url: receiverUrl, events: ["alert.opened"] },
]) {
assert.equal((await send("POST", "/api/event-webhooks", bad)).status, 400, JSON.stringify(bad));
}
who = "operator";
assert.equal((await send("GET", "/api/event-webhooks")).status, 403);
who = "admin";
});
test("an event goes to the webhooks that want it, signed so the receiver can check it", async () => {
received.length = 0;
emitEvent("alert.opened", { alert: { key: "offline:server:1", severity: "critical", message: "elsa hasn't reported" } });
emitEvent("alert.resolved", { alert: { key: "x", message: "not wanted by this webhook" } });
await waitFor(1);
await new Promise((r) => setTimeout(r, 150));
assert.equal(received.length, 1, "only the event it asked for");
const got = received[0];
assert.equal(got.headers["x-hlm-event"], "alert.opened");
assert.equal(got.headers["x-hlm-signature"], `sha256=${createHmac("sha256", HOOK_SECRET).update(got.body).digest("hex")}`);
assert.equal(got.headers["x-hlm-signature"], signBody(HOOK_SECRET, got.body));
const body = JSON.parse(got.body);
assert.equal(body.event, "alert.opened");
assert.equal(body.source, "homelab-manager");
assert.equal(body.data.alert.message, "elsa hasn't reported");
assert.equal(body.id, got.headers["x-hlm-delivery"]);
const [row] = await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, hookId));
assert.equal(row.lastResult, "200");
assert.ok(row.lastAttemptAt);
});
test("a switched-off webhook hears nothing", async () => {
assert.equal((await send("PATCH", `/api/event-webhooks/${hookId}`, { enabled: false })).status, 200);
received.length = 0;
emitEvent("alert.opened", { alert: { message: "x" } });
await new Promise((r) => setTimeout(r, 300));
assert.equal(received.length, 0);
await send("PATCH", `/api/event-webhooks/${hookId}`, { enabled: true });
});
test("a delivery that fails is tried once more, and what happened is noted", async () => {
const [row] = await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, hookId));
received.length = 0;
receiverStatus = [500, 200];
const ok = await deliver(row, "alert.opened", { n: 1 }, { retryAfterMs: 10 });
assert.deepEqual(ok, { ok: true, result: "200" });
assert.equal(received.length, 2, "tried twice");
assert.equal(received[0].headers["x-hlm-delivery"], received[1].headers["x-hlm-delivery"], "the same message both times");
receiverStatus = [503, 503];
const bad = await deliver(row, "alert.opened", { n: 2 }, { retryAfterMs: 10 });
assert.deepEqual(bad, { ok: false, result: "503" });
assert.equal((await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, hookId)))[0].lastResult, "503");
const dead = await deliver({ ...row, url: "http://127.0.0.1:1/nothing" }, "alert.opened", {}, { retryAfterMs: 10 });
assert.equal(dead.ok, false);
});
test("a test message goes out whatever the webhook is set to, and says how it went", async () => {
received.length = 0;
receiverStatus = [];
const res = await send("POST", `/api/event-webhooks/${hookId}/test`);
assert.deepEqual(await res.json(), { ok: true, result: "200" });
assert.equal(received[0].headers["x-hlm-event"], "test");
assert.equal((await send("POST", "/api/event-webhooks/99999/test")).status, 404);
});
test("maintenance beginning and ending is noticed, however it came about, and a first look says nothing", async () => {
resetMaintenanceEventsForTests();
const [elsa] = await db.select().from(servers).where(eq(servers.name, 'elsa "main"'));
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: elsa.id, reason: "before", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString(), createdBy: "Ada" });
assert.deepEqual(await checkMaintenanceEvents(), { started: [], ended: [] }, "already running when first looked at: not announced");
const [w] = await db.insert(maintenanceWindows).values({ targetType: "server", targetId: elsa.id, reason: "patching", startedAt: new Date(Date.now() - 1000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString(), createdBy: "Ada", source: "schedule:1:2026-10-17" }).returning();
received.length = 0;
const started = await checkMaintenanceEvents();
assert.deepEqual(started, { started: [w.id], ended: [] });
await waitFor(1);
assert.equal(received[0].headers["x-hlm-event"], "maintenance.started");
const payload = JSON.parse(received[0].body).data.window;
assert.equal(payload.targetName, 'elsa "main"');
assert.equal(payload.reason, "patching");
assert.equal(payload.source, "schedule:1:2026-10-17");
// ended early
await db.update(maintenanceWindows).set({ endsAt: new Date(Date.now() - 500).toISOString() }).where(eq(maintenanceWindows.id, w.id));
assert.deepEqual(await checkMaintenanceEvents(), { started: [], ended: [w.id] });
assert.deepEqual(await checkMaintenanceEvents(), { started: [], ended: [] }, "and only once");
});
test("a webhook can be removed", async () => {
assert.equal((await send("DELETE", `/api/event-webhooks/${hookId}`)).status, 204);
assert.equal((await send("DELETE", `/api/event-webhooks/${hookId}`)).status, 404);
});
});