/** * What an agent could not collect and what to do about it, and the services a server is told to ignore. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; import type { Server } from "node:http"; const dir = mkdtempSync(join(tmpdir(), "hm-diag-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { describeDiagnostics, parseDiagnostics } = await import("../src/services/agentDiagnostics.js"); const { isServiceWatched, onlyWatchedServices, markWatchedServices } = await import("../src/services/hostHealth.js"); const { checkThresholds, parseThresholds } = await import("../src/services/serverThresholds.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { buildMetrics } = await import("../src/services/metrics.js"); const { serversRouter } = await import("../src/routes/servers.js"); const { agentReportRouter } = await import("../src/routes/agentReport.js"); const { hashToken } = await import("../src/services/tokens.js"); await runMigrations(); const latest = { linux: 6, windows: 5 }; const ctx = (extra: Partial<{ osType: string; lastSeenAt: string | null; agentVersion: number | null }> = {}) => ({ osType: "linux", lastSeenAt: "2026-10-18T10:00:00Z", agentVersion: 6, latest, ...extra }); const stored = (items: unknown[]) => JSON.stringify({ items, reportedAt: "2026-10-18T10:00:00Z" }); describe("saying what to do about what couldn't be collected", () => { test("a missing tool is a tip with the fix; ZFS that isn't there is not worth saying", () => { const notes = describeDiagnostics( parseDiagnostics(stored([ { check: "smart", status: "unavailable", reason: "tool_missing", detail: "smartctl" }, { check: "zfs", status: "unavailable", reason: "tool_missing", detail: "zpool" }, { check: "hardware", status: "unavailable", reason: "not_found" }, ])), ctx(), ); assert.deepEqual(notes.map((n) => n.check), ["smart", "hardware"], "no word about ZFS"); assert.equal(notes[0].level, "info"); assert.match(notes[0].fix!, /smartmontools/); }); test("a failure is a warning that says what the agent said, with a fix for the system it is on", () => { const failed = stored([{ check: "updates", status: "failed", reason: "error", detail: "apt did not answer" }]); const linux = describeDiagnostics(parseDiagnostics(failed), ctx())[0]; assert.equal(linux.level, "warning"); assert.equal(linux.detail, "apt did not answer"); assert.match(linux.fix!, /--check-updates/); const windows = describeDiagnostics(parseDiagnostics(failed), ctx({ osType: "windows", agentVersion: 5 }))[0]; assert.match(windows.fix!, /Windows Update/); }); test("a step that didn't run is a warning, and warnings come first", () => { const notes = describeDiagnostics(parseDiagnostics(stored([{ check: "hardware", status: "unavailable", reason: "not_found" }, { check: "prepare", status: "unavailable", reason: "step_missing" }])), ctx()); assert.deepEqual(notes.map((n) => [n.check, n.level]), [["prepare", "warning"], ["hardware", "info"]]); }); test("something it has no words for is still said, rather than dropped", () => { const notes = describeDiagnostics(parseDiagnostics(stored([{ check: "gpu_temps", status: "failed", reason: "error" }, { check: "fans", status: "unavailable", reason: "no_sensor" }])), ctx()); assert.equal(notes[0].level, "warning"); assert.match(notes[0].title, /gpu_temps/); assert.match(notes[1].title, /fans isn't available \(no sensor\)/); }); test("an agent that is behind is told so, whether or not it said anything", () => { const notes = describeDiagnostics(null, ctx({ agentVersion: 3 })); assert.equal(notes.length, 1); assert.match(notes[0].title, /version 3; 6 is current/); assert.deepEqual(describeDiagnostics(null, ctx()), [], "an agent that is current and silent has nothing to say"); assert.deepEqual(describeDiagnostics(null, ctx({ lastSeenAt: null, agentVersion: 3 })), [], "a server that has never reported has no agent to update"); }); test("what is stored is read leniently", () => { assert.equal(parseDiagnostics(null), null); assert.equal(parseDiagnostics("not json"), null); const read = parseDiagnostics(stored([{ check: "ok", status: "fine" }, "x", { check: "smart", status: "failed", detail: "a\nb" }]))!; assert.deepEqual(read.items, [{ check: "smart", status: "failed", detail: "a b" }]); }); }); describe("services a server is told to watch", () => { test("names and patterns, in any case, with or without .service", () => { assert.equal(isServiceWatched("gupdate.service", ["gupdate"]), true); assert.equal(isServiceWatched("GUpdate", ["gupdate*"]), true); assert.equal(isServiceWatched("snap.lxd.activate.service", ["snap.*"]), true); assert.equal(isServiceWatched("nginx.service", ["gupdate*", "snap.*"]), false); assert.equal(isServiceWatched("nginx.service", ["nginx.service"]), true); assert.equal(isServiceWatched("mynginx.service", ["nginx"]), false, "a name is the whole name"); assert.equal(isServiceWatched("a+b.service", ["a+b"]), true, "characters that mean something to a regex are only characters here"); assert.equal(isServiceWatched("anything", []), false); assert.equal(isServiceWatched("anything", undefined), false); }); test("* is every service", () => { assert.equal(isServiceWatched("anything.service", ["*"]), true); }); test("only the watched are kept for what is judged, all are shown marked", () => { const health = { failedUnits: [{ unit: "nginx.service", description: "" }, { unit: "gupdate", description: "Google Update" }], reportedAt: "t" }; assert.deepEqual(onlyWatchedServices(health, ["nginx"])!.failedUnits!.map((u) => u.unit), ["nginx.service"]); assert.deepEqual(markWatchedServices(health, ["nginx"])!.failedUnits!.map((u) => [u.unit, u.watched]), [["nginx.service", true], ["gupdate", false]]); assert.equal(onlyWatchedServices(null, ["x"]), null); assert.deepEqual(onlyWatchedServices(health, [])!.failedUnits, [], "nothing is watched until it is said: no service counts as failed"); assert.deepEqual(onlyWatchedServices(health, undefined)!.failedUnits, []); assert.deepEqual(markWatchedServices(health, [])!.failedUnits!.map((u) => u.watched), [false, false]); const silent = { reportedAt: "t" }; assert.deepEqual(onlyWatchedServices(silent, []), silent, "a report with nothing about services is left as it is"); }); test("the list is part of a server's own thresholds: kept tidy, and a wrong one is refused", () => { const ok = checkThresholds({ watchServices: [" gupdate* ", "gupdate*", "", "snap.*"] }); assert.deepEqual(ok.ok && ok.value, { watchServices: ["gupdate*", "snap.*"] }); assert.equal(checkThresholds({ watchServices: "gupdate" }).ok, false); assert.equal(checkThresholds({ watchServices: [3] }).ok, false); assert.equal(checkThresholds({ watchServices: ["x".repeat(101)] }).ok, false); assert.equal(checkThresholds({ watchServices: Array.from({ length: 51 }, (_, i) => `s${i}`) }).ok, false); assert.equal(checkThresholds({ watchServices: [] }).ok && (checkThresholds({ watchServices: [] }) as { value: unknown }).value, null, "an empty list is no thresholds at all"); assert.deepEqual(parseThresholds(JSON.stringify({ watchServices: ["a", 3, "", "a"] })), { watchServices: ["a"] }); }); }); describe("through a report, the page, the alerts and the figures", () => { let app: Server; let base = ""; let id = 0; before(async () => { await db.insert(users).values([{ oidcSub: "admin", name: "Ada", role: "admin" }]); const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: new Date().toISOString(), agentVersion: 8 }).returning(); id = s.id; const a = express(); a.use("/api/agent/report", agentReportRouter); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: "admin" } }; next(); }); a.use("/api/servers", serversRouter); app = await new Promise((resolve) => { const x = a.listen(0, "127.0.0.1", () => resolve(x)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); const report = (system: unknown) => fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", agent_version: 8, tasks: [], system }) }); const detail = async () => await (await fetch(`${base}/servers/${id}/detail`)).json(); const patch = (body: unknown) => fetch(`${base}/servers/${id}`, { method: "PATCH", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); test("what the agent says it couldn't collect is shown with what to do; garbage entries are dropped; silence keeps what was said", async () => { assert.equal( (await report({ diagnostics: [{ check: "smart", status: "unavailable", reason: "tool_missing", detail: "smartctl" }, { check: "x", status: "bogus" }, "junk", { status: "failed" }] })).status, 202, ); let d = (await detail()).diagnostics; assert.deepEqual(d.map((n: { check: string }) => n.check), ["smart"]); assert.match(d[0].fix, /smartmontools/); assert.equal((await report({ cpu: { cores: 2 } })).status, 202); assert.equal((await detail()).diagnostics.length, 1, "a report that doesn't say keeps what was said"); assert.equal((await report({ diagnostics: "garbage" })).status, 202); assert.equal((await detail()).diagnostics.length, 1, "and so does one that isn't a list"); assert.equal((await report({ diagnostics: [] })).status, 202); assert.deepEqual((await detail()).diagnostics, [], "an empty list says nothing was missed"); const row = (await (await fetch(`${base}/servers`)).json()).servers.find((x: { id: number }) => x.id === id); assert.equal(row.agentDiagnostics, undefined, "never as raw stored text"); }); test("nothing is an alert until it is watched; a watched service that has failed is, and a failed one that is not watched still shows on the page", async () => { assert.equal((await report({ health: { failed_units: [{ unit: "nginx.service", description: "web" }, { unit: "gupdate.service", description: "Google Update" }] } })).status, 202); invalidateAlertsCache(); const failing = async () => (await getAlerts(true)).alerts.find((a) => a.id.startsWith("services:failed-units") && a.id.includes(`:server:${id}`)); assert.equal(await failing(), undefined, "nothing is watched, so nothing is an alert"); assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\{server="elsa"[^}]*\} 0`)); assert.deepEqual((await detail()).server.hostHealth.failedUnits.map((u: { unit: string; watched?: boolean }) => [u.unit, u.watched]), [["nginx.service", false], ["gupdate.service", false]], "still shown, marked not watched"); assert.equal((await patch({ thresholds: { watchServices: ["nginx"] } })).status, 200); assert.match((await failing())!.message, /1 service has failed — nginx\.service/, "the alert cache was dropped when the list changed"); assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\{server="elsa"[^}]*\} 1`)); const units = (await detail()).server.hostHealth.failedUnits; assert.deepEqual(units.map((u: { unit: string; watched?: boolean }) => [u.unit, u.watched]), [["nginx.service", true], ["gupdate.service", false]]); assert.equal((await patch({ thresholds: { watchServices: ["*"] } })).status, 200); assert.match((await failing())!.message, /2 services have failed/, "* watches them all, as it always was"); assert.equal((await patch({ thresholds: null })).status, 200); assert.equal(await failing(), undefined, "taking the list away watches nothing again"); }); test("a list that isn't one is refused with a message", async () => { const res = await patch({ thresholds: { watchServices: "gupdate" } }); assert.equal(res.status, 400); assert.match((await res.json()).message, /should be a list/); }); });