/** * The Prometheus endpoint and its tokens, and the signed event webhooks. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import http from "node:http"; import { createHmac } from "node:crypto"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-metrics-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers, secrets, maintenanceWindows, metricsTokens, eventWebhooks, auditLog } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { metricsRouter, metricsTokensRouter } = await import("../src/routes/metrics.js"); const { eventWebhooksRouter } = await import("../src/routes/eventWebhooks.js"); const { escapeLabel } = await import("../src/services/metrics.js"); const { deliver, emitEvent, signBody } = await import("../src/services/eventWebhooks.js"); const { checkMaintenanceEvents, resetMaintenanceEventsForTests } = await import("../src/services/maintenanceEvents.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); describe("escaping", () => { test("a label value keeps its quotes, backslashes and new lines from breaking the line", () => { assert.equal(escapeLabel('say "hi"'), 'say \\"hi\\"'); assert.equal(escapeLabel("C:\\temp"), "C:\\\\temp"); assert.equal(escapeLabel("a\nb"), "a\\nb"); }); }); describe("metrics and webhooks", () => { let app: http.Server; let base = ""; let who = "admin"; let receiver: http.Server; const received: { headers: http.IncomingHttpHeaders; body: string }[] = []; let receiverStatus: number[] = []; // statuses to answer with, in turn; 200 once it runs out let receiverUrl = ""; before(async () => { await db.insert(users).values([ { oidcSub: "admin", name: "Ada", role: "admin" }, { oidcSub: "operator", name: "Olga", role: "operator" }, ]); const now = new Date(); await db.insert(servers).values([ { name: 'elsa "main"', apiTokenHash: "a", apiTokenPrefix: "a", lastSeenAt: now.toISOString(), cpuLoadPercent: 12.5, uptimeSeconds: 86_400, memTotalBytes: 1000, memUsedBytes: 250, disks: JSON.stringify([{ mount: "C:\\", sizeBytes: 100, usedBytes: 40 }]), pendingUpdates: JSON.stringify({ manager: "apt", count: 7, security: 2, rebootRequired: true, reportedAt: now.toISOString() }), agentVersion: 1, }, { name: "anna", apiTokenHash: "b", apiTokenPrefix: "b", lastSeenAt: new Date(now.getTime() - 5 * 3600_000).toISOString() }, { name: "never", apiTokenHash: "c", apiTokenPrefix: "c" }, ]); await db.insert(secrets).values({ name: "VPN key", type: "api_token", expiryDate: new Date(now.getTime() + 9 * 86_400_000).toISOString().slice(0, 10), warnDays: 30 }); receiver = await new Promise((resolve) => { const s = http.createServer((req, res) => { let raw = ""; req.on("data", (c) => (raw += c)); req.on("end", () => { received.push({ headers: req.headers, body: raw }); res.statusCode = receiverStatus.shift() ?? 200; res.end("ok"); }); }); s.listen(0, "127.0.0.1", () => resolve(s)); }); receiverUrl = `http://127.0.0.1:${(receiver.address() as AddressInfo).port}/hook`; const a = express(); a.use("/metrics", metricsRouter); // no session: it is the bearer token that counts const api = express(); api.use(express.json()); api.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); api.use("/api/metrics-tokens", metricsTokensRouter); api.use("/api/event-webhooks", eventWebhooksRouter); a.use(api); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`; }); after(() => { app.close(); receiver.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); let secret = ""; let tokenId = 0; test("an administrator makes a token; the secret is shown once and only its hash is kept", async () => { const res = await send("POST", "/api/metrics-tokens", { label: "Prometheus" }); assert.equal(res.status, 201); const body = await res.json(); secret = body.secret; tokenId = body.token.id; assert.match(secret, /^hlm_m_/); const list = await (await send("GET", "/api/metrics-tokens")).json(); assert.ok(!JSON.stringify(list).includes(secret) && !JSON.stringify(list).includes("tokenHash")); const [stored] = await db.select().from(metricsTokens); assert.notEqual(stored.tokenHash, secret); assert.equal(stored.tokenHash.length, 64); who = "operator"; assert.equal((await send("POST", "/api/metrics-tokens", { label: "x" })).status, 403); assert.equal((await send("GET", "/api/metrics-tokens")).status, 403); who = "admin"; assert.equal((await send("POST", "/api/metrics-tokens", { label: "" })).status, 400); }); test("/metrics wants the token in a header: nothing, a wrong one, or one in the address all get the same refusal", async () => { const none = await fetch(`${base}/metrics`); assert.equal(none.status, 401); assert.match(none.headers.get("www-authenticate") ?? "", /Bearer/); const refusal = await none.text(); assert.equal((await fetch(`${base}/metrics`, { headers: { authorization: "Bearer hlm_m_wrongwrongwrongwrongwrong" } })).status, 401); assert.equal((await fetch(`${base}/metrics?token=${secret}`)).status, 401, "a token in the address is not accepted"); assert.equal(await (await fetch(`${base}/metrics`, { headers: { authorization: "Bearer nope" } })).text(), refusal); }); test("with the token it answers in Prometheus's format, with the servers, alerts and the rest", async () => { const res = await fetch(`${base}/metrics`, { headers: { authorization: `Bearer ${secret}` } }); assert.equal(res.status, 200); assert.match(res.headers.get("content-type") ?? "", /^text\/plain; version=0\.0\.4/); assert.equal(res.headers.get("cache-control"), "no-store"); const text = await res.text(); assert.match(text, /# TYPE homelab_server_up gauge/); assert.match(text, /homelab_server_up\{server="elsa \\"main\\""\} 1/, "the name's quotes are escaped"); assert.match(text, /homelab_server_up\{server="anna"\} 0/, "five hours silent is past the offline threshold"); assert.ok(!text.includes('server="never"'), "a server that never reported has nothing to say"); assert.match(text, /homelab_server_cpu_load_percent\{server="elsa \\"main\\""\} 12\.5/); assert.match(text, /homelab_server_memory_used_percent\{server="elsa \\"main\\""\} 25/); const boot = /homelab_server_boot_time_seconds\{server="elsa \\"main\\""\} (\d+)/.exec(text); assert.ok(boot, "a server that says how long it has been up gets a boot time"); assert.ok(Math.abs(Number(boot[1]) - (Math.floor(Date.now() / 1000) - 86_400)) < 120, "the moment of its last report minus the uptime"); assert.ok(!text.includes('homelab_server_boot_time_seconds{server="anna"}'), "one that does not say gets none"); assert.match(text, /homelab_server_disk_used_percent\{server="elsa \\"main\\"",mount="C:\\\\"\} 40/, "a backslash in a mount is escaped"); assert.match(text, /homelab_server_pending_updates\{server="elsa \\"main\\""\} 7/); assert.match(text, /homelab_server_pending_security_updates\{server="elsa \\"main\\""\} 2/); assert.match(text, /homelab_server_reboot_required\{server="elsa \\"main\\""\} 1/); assert.match(text, /homelab_server_agent_outdated\{server="elsa \\"main\\""\} 1/, "version 1 is behind"); assert.match(text, /homelab_alerts\{severity="critical"\} \d+/); assert.match(text, /homelab_secret_expiry_days\{name="VPN key"\} 9/); assert.match(text, /homelab_maintenance_active_windows 0/); assert.match(text, /homelab_scrape_duration_seconds /); }); test("every line is valid exposition format, and each metric is described once", async () => { const text = await (await fetch(`${base}/metrics`, { headers: { authorization: `Bearer ${secret}` } })).text(); const helps: string[] = []; for (const line of text.trim().split("\n")) { if (line.startsWith("# HELP ")) { helps.push(line.split(" ")[2]); continue; } if (line.startsWith("# TYPE ")) continue; assert.match(line, /^[a-zA-Z_:][a-zA-Z0-9_:]*(\{([a-zA-Z_][a-zA-Z0-9_]*="([^"\\]|\\.)*",?)*\})? -?\d+(\.\d+)?$/, line); } assert.equal(new Set(helps).size, helps.length, "no metric is declared twice"); }); test("use is noted, and a removed token stops working at once", async () => { const [row] = await db.select().from(metricsTokens).where(eq(metricsTokens.id, tokenId)); assert.ok(row.lastUsedAt); assert.equal((await send("DELETE", `/api/metrics-tokens/${tokenId}`)).status, 204); assert.equal((await fetch(`${base}/metrics`, { headers: { authorization: `Bearer ${secret}` } })).status, 401); assert.equal((await send("DELETE", `/api/metrics-tokens/${tokenId}`)).status, 404); const entries = (await db.select().from(auditLog)).filter((e) => e.category === "metrics_token"); assert.deepEqual(entries.map((e) => e.action), ["create", "delete"]); assert.ok(entries.every((e) => !(e.detail ?? "").includes(secret))); }); // ── event webhooks ── let hookId = 0; const HOOK_SECRET = "s3cret-for-signing"; const waitFor = async (n: number, ms = 3000) => { for (let t = 0; t < ms && received.length < n; t += 25) await new Promise((r) => setTimeout(r, 25)); }; test("a webhook is made with a secret that is never shown again, and kept encrypted", async () => { const res = await send("POST", "/api/event-webhooks", { label: "Home Assistant", url: receiverUrl, secret: HOOK_SECRET, events: ["alert.opened", "maintenance.started"] }); assert.equal(res.status, 201); const hook = (await res.json()).webhook; hookId = hook.id; assert.equal(hook.hasSecret, true); assert.deepEqual(hook.events, ["alert.opened", "maintenance.started"]); assert.ok(!JSON.stringify(await (await send("GET", "/api/event-webhooks")).json()).includes(HOOK_SECRET)); const [stored] = await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, hookId)); assert.match(stored.secret ?? "", /^enc:v1:/); const audit = (await db.select().from(auditLog)).find((e) => e.category === "webhook" && e.action === "create"); assert.ok(!(audit?.detail ?? "").includes(receiverUrl) && !(audit?.detail ?? "").includes(HOOK_SECRET), "neither the address nor the secret goes in the audit log"); }); test("a bad webhook is refused: not http, a password in the address, no events, only admins", async () => { for (const bad of [ { label: "x", url: "ftp://x.example/hook", events: ["alert.opened"] }, { label: "x", url: "https://user:pass@x.example/hook", events: ["alert.opened"] }, { label: "x", url: "not a url", events: ["alert.opened"] }, { label: "x", url: receiverUrl, events: [] }, { label: "x", url: receiverUrl, events: ["alert.exploded"] }, { label: "", url: receiverUrl, events: ["alert.opened"] }, ]) { assert.equal((await send("POST", "/api/event-webhooks", bad)).status, 400, JSON.stringify(bad)); } who = "operator"; assert.equal((await send("GET", "/api/event-webhooks")).status, 403); who = "admin"; }); test("an event goes to the webhooks that want it, signed so the receiver can check it", async () => { received.length = 0; emitEvent("alert.opened", { alert: { key: "offline:server:1", severity: "critical", message: "elsa hasn't reported" } }); emitEvent("alert.resolved", { alert: { key: "x", message: "not wanted by this webhook" } }); await waitFor(1); await new Promise((r) => setTimeout(r, 150)); assert.equal(received.length, 1, "only the event it asked for"); const got = received[0]; assert.equal(got.headers["x-hlm-event"], "alert.opened"); assert.equal(got.headers["x-hlm-signature"], `sha256=${createHmac("sha256", HOOK_SECRET).update(got.body).digest("hex")}`); assert.equal(got.headers["x-hlm-signature"], signBody(HOOK_SECRET, got.body)); const body = JSON.parse(got.body); assert.equal(body.event, "alert.opened"); assert.equal(body.source, "homelab-manager"); assert.equal(body.data.alert.message, "elsa hasn't reported"); assert.equal(body.id, got.headers["x-hlm-delivery"]); const [row] = await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, hookId)); assert.equal(row.lastResult, "200"); assert.ok(row.lastAttemptAt); }); test("a switched-off webhook hears nothing", async () => { assert.equal((await send("PATCH", `/api/event-webhooks/${hookId}`, { enabled: false })).status, 200); received.length = 0; emitEvent("alert.opened", { alert: { message: "x" } }); await new Promise((r) => setTimeout(r, 300)); assert.equal(received.length, 0); await send("PATCH", `/api/event-webhooks/${hookId}`, { enabled: true }); }); test("a delivery that fails is tried once more, and what happened is noted", async () => { const [row] = await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, hookId)); received.length = 0; receiverStatus = [500, 200]; const ok = await deliver(row, "alert.opened", { n: 1 }, { retryAfterMs: 10 }); assert.deepEqual(ok, { ok: true, result: "200" }); assert.equal(received.length, 2, "tried twice"); assert.equal(received[0].headers["x-hlm-delivery"], received[1].headers["x-hlm-delivery"], "the same message both times"); receiverStatus = [503, 503]; const bad = await deliver(row, "alert.opened", { n: 2 }, { retryAfterMs: 10 }); assert.deepEqual(bad, { ok: false, result: "503" }); assert.equal((await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, hookId)))[0].lastResult, "503"); const dead = await deliver({ ...row, url: "http://127.0.0.1:1/nothing" }, "alert.opened", {}, { retryAfterMs: 10 }); assert.equal(dead.ok, false); }); test("a test message goes out whatever the webhook is set to, and says how it went", async () => { received.length = 0; receiverStatus = []; const res = await send("POST", `/api/event-webhooks/${hookId}/test`); assert.deepEqual(await res.json(), { ok: true, result: "200" }); assert.equal(received[0].headers["x-hlm-event"], "test"); assert.equal((await send("POST", "/api/event-webhooks/99999/test")).status, 404); }); test("maintenance beginning and ending is noticed, however it came about, and a first look says nothing", async () => { resetMaintenanceEventsForTests(); const [elsa] = await db.select().from(servers).where(eq(servers.name, 'elsa "main"')); await db.insert(maintenanceWindows).values({ targetType: "server", targetId: elsa.id, reason: "before", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString(), createdBy: "Ada" }); assert.deepEqual(await checkMaintenanceEvents(), { started: [], ended: [] }, "already running when first looked at: not announced"); const [w] = await db.insert(maintenanceWindows).values({ targetType: "server", targetId: elsa.id, reason: "patching", startedAt: new Date(Date.now() - 1000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString(), createdBy: "Ada", source: "schedule:1:2026-10-17" }).returning(); received.length = 0; const started = await checkMaintenanceEvents(); assert.deepEqual(started, { started: [w.id], ended: [] }); await waitFor(1); assert.equal(received[0].headers["x-hlm-event"], "maintenance.started"); const payload = JSON.parse(received[0].body).data.window; assert.equal(payload.targetName, 'elsa "main"'); assert.equal(payload.reason, "patching"); assert.equal(payload.source, "schedule:1:2026-10-17"); // ended early await db.update(maintenanceWindows).set({ endsAt: new Date(Date.now() - 500).toISOString() }).where(eq(maintenanceWindows.id, w.id)); assert.deepEqual(await checkMaintenanceEvents(), { started: [], ended: [w.id] }); assert.deepEqual(await checkMaintenanceEvents(), { started: [], ended: [] }, "and only once"); }); test("a webhook can be removed", async () => { assert.equal((await send("DELETE", `/api/event-webhooks/${hookId}`)).status, 204); assert.equal((await send("DELETE", `/api/event-webhooks/${hookId}`)).status, 404); }); });