/** * What an agent can say about a machine's health (failed services, disk SMART, ZFS pools), the alerts and figures that come from it, and * the switch that lets agents that are behind replace themselves. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; import type { Server } from "node:http"; const dir = mkdtempSync(join(tmpdir(), "hm-hosthealth-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers, maintenanceWindows } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { hostHealthConditions, nextHostHealth, parseHostHealth } = await import("../src/services/hostHealth.js"); const { evaluateHealth } = await import("../src/services/healthMonitor.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { buildMetrics } = await import("../src/services/metrics.js"); const { latestAgentSha256, latestAgentVersions } = await import("../src/services/agentVersions.js"); const { serversRouter } = await import("../src/routes/servers.js"); const { agentReportRouter } = await import("../src/routes/agentReport.js"); const { hashToken } = await import("../src/services/tokens.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); const NOW = "2026-10-17T12:00:00.000Z"; const health = (extra: object = {}) => ({ reportedAt: NOW, ...extra }); describe("what a health report amounts to", () => { test("failed services, a failing disk, a worn one and a pool that isn't online are each a problem; the healthy are not", () => { const conditions = hostHealthConditions(7, "elsa", { reportedAt: NOW, failedUnits: [{ unit: "nginx.service", description: "" }, { unit: "backup.service", description: "" }], smart: [ { device: "/dev/sda", model: "WD Red", healthy: false, temperatureC: 40, wearPercent: null }, { device: "/dev/nvme0", model: "Samsung", healthy: true, temperatureC: 38, wearPercent: 93 }, { device: "/dev/sdb", model: "Fine", healthy: true, temperatureC: 31, wearPercent: 12 }, { device: "/dev/sdc", model: "No verdict", healthy: null, temperatureC: null, wearPercent: null }, ], zfs: [{ name: "tank", health: "DEGRADED", capacityPercent: 40 }, { name: "rpool", health: "ONLINE", capacityPercent: 10 }], }); const byKey = Object.fromEntries(conditions.map((c) => [c.key, c])); assert.deepEqual(Object.keys(byKey).sort(), ["failed-units:server:7", "smart:server:7:/dev/nvme0", "smart:server:7:/dev/sda", "zfs:server:7:tank"]); assert.equal(byKey["failed-units:server:7"].severity, "warning"); assert.match(byKey["failed-units:server:7"].message, /2 services have failed — nginx\.service, backup\.service/); assert.equal(byKey["smart:server:7:/dev/sda"].severity, "critical"); assert.match(byKey["smart:server:7:/dev/sda"].message, /failing/); assert.equal(byKey["smart:server:7:/dev/nvme0"].severity, "warning"); assert.match(byKey["smart:server:7:/dev/nvme0"].message, /93% of its rated life/); assert.equal(byKey["zfs:server:7:tank"].severity, "critical"); assert.match(byKey["zfs:server:7:tank"].message, /DEGRADED/); }); test("nothing said, or everything fine, is no problem; a long list of services is shortened", () => { assert.deepEqual(hostHealthConditions(1, "x", null), []); assert.deepEqual(hostHealthConditions(1, "x", health({ failedUnits: [], smart: [], zfs: [] })), []); const many = Array.from({ length: 9 }, (_, i) => ({ unit: `u${i}.service`, description: "" })); assert.match(hostHealthConditions(1, "x", health({ failedUnits: many }))[0].message, /and 5 more/); assert.match(hostHealthConditions(1, "x", health({ failedUnits: [many[0]] }))[0].message, /1 service has failed/); }); test("a report replaces the parts it carries and keeps the parts it doesn't", () => { const first = nextHostHealth(null, { failed_units: [{ unit: "a.service" }], zfs: [{ name: "tank", health: "ONLINE", capacity_percent: 20 }] }, "t1"); assert.equal(parseHostHealth(first)!.failedUnits![0].unit, "a.service"); // the next report knows about disks only const second = nextHostHealth(first, { smart: [{ device: "/dev/sda", healthy: true }] }, "t2"); const merged = parseHostHealth(second)!; assert.equal(merged.failedUnits![0].unit, "a.service", "services kept"); assert.equal(merged.zfs![0].name, "tank", "pools kept"); assert.equal(merged.smart![0].device, "/dev/sda"); assert.equal(merged.reportedAt, "t2"); // an empty list is an answer ("none failed"), and clears the old one assert.deepEqual(parseHostHealth(nextHostHealth(second, { failed_units: [] }, "t3"))!.failedUnits, []); // nothing in the report: what was stored is untouched assert.equal(nextHostHealth(second, {}, "t4"), second); assert.equal(parseHostHealth("not json"), null); }); test("an offline server isn't judged on its health, and an online one is", () => { const snap = (lastSeenAt: string) => ({ servers: [{ id: 3, name: "elsa", lastSeenAt, disks: [], health: health({ failedUnits: [{ unit: "a.service", description: "" }] }) }], proxmox: [], synology: [] }); const now = Date.parse(NOW); const limits = { serverOfflineMinutes: 60, diskUsagePercent: 90 }; const online = evaluateHealth(snap(new Date(now - 60_000).toISOString()), limits, now).map((c) => c.key); assert.deepEqual(online, ["failed-units:server:3"]); const offline = evaluateHealth(snap(new Date(now - 5 * 3600_000).toISOString()), limits, now).map((c) => c.key); assert.deepEqual(offline, ["offline:server:3"]); }); }); describe("through the report, the alerts, the figures and the page", () => { let app: Server; let base = ""; let id = 0; let who = "admin"; before(async () => { await db.insert(users).values([ { oidcSub: "admin", name: "Ada", role: "admin" }, { oidcSub: "operator", name: "Olle", role: "operator" }, ]); const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", thresholds: JSON.stringify({ watchServices: ["*"] }) }).returning(); id = s.id; const a = express(); a.use("/api/agent/report", agentReportRouter); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/servers", serversRouter); app = await new Promise((resolve) => { const x = a.listen(0, "127.0.0.1", () => resolve(x)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); const report = (system: unknown, extra: Record = {}) => fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", tasks: [], system, ...extra }) }); const detail = async () => (await (await fetch(`${base}/servers/${id}/detail`)).json()).server; test("what an agent says is stored and shown; a garbled part costs the report nothing", async () => { const res = await report({ health: { failed_units: [{ unit: "nginx.service", description: "A high performance web server" }], smart: [{ device: "/dev/sda", model: "WD Red", healthy: false, temperature_c: 41, wear_percent: null }], zfs: "this isn't a list", }, }); assert.equal(res.status, 202); let s = await detail(); assert.equal(s.hostHealth.failedUnits[0].unit, "nginx.service"); assert.equal(s.hostHealth.smart[0].healthy, false); assert.equal(s.hostHealth.zfs, undefined, "the part that made no sense was dropped"); // the list endpoint shows it too, and never as raw stored text const row = (await (await fetch(`${base}/servers`)).json()).servers.find((x: { id: number }) => x.id === id); assert.equal(typeof row.hostHealth, "object"); // a report with no health at all leaves it as it was; garbage in place of the whole thing too assert.equal((await report({ cpu: { cores: 2 } })).status, 202); assert.equal((await report({ health: 12 })).status, 202); s = await detail(); assert.equal(s.hostHealth.failedUnits[0].unit, "nginx.service"); }); test("they appear on the Alerts page as their own kind, under the server, and a maintenance window silences them", async () => { invalidateAlertsCache(); let alerts = (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`)); const services = alerts.find((a) => a.id.startsWith("services:failed-units")); assert.ok(services, JSON.stringify(alerts.map((a) => a.id))); assert.equal(services.category, "services"); assert.equal(services.link, `/servers/${id}?view=health`, "it opens the Health section of the server page"); assert.equal(services.silenced, false); const disk = alerts.find((a) => a.id.startsWith("disk:smart")); assert.ok(disk && disk.severity === "critical", "a failing disk is critical, in the disk category"); await db.insert(maintenanceWindows).values({ targetType: "server", targetId: id, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() }); invalidateAlertsCache(); alerts = (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`)); assert.ok(alerts.length >= 2 && alerts.every((a) => a.silenced), "all held back under maintenance"); await db.delete(maintenanceWindows).where(eq(maintenanceWindows.targetId, id)); // the services come right once the agent says none have failed await report({ health: { failed_units: [] } }); invalidateAlertsCache(); assert.equal((await getAlerts(true)).alerts.some((a) => a.id.startsWith("services:")), false); }); test("/metrics carries them", async () => { await report({ health: { failed_units: [{ unit: "a.service" }, { unit: "b.service" }], zfs: [{ name: "tank", health: "DEGRADED", capacity_percent: 12 }], smart: [{ device: "/dev/sda", healthy: true, temperature_c: 33, wear_percent: 5 }] } }); const text = await buildMetrics(); assert.match(text, /homelab_server_failed_units\{server="elsa"\} 2/); assert.match(text, /homelab_server_zfs_pool_online\{server="elsa",pool="tank"\} 0/); assert.match(text, /homelab_server_disk_healthy\{server="elsa",device="\/dev\/sda"\} 1/); assert.match(text, /homelab_server_disk_wear_percent\{server="elsa",device="\/dev\/sda"\} 5/); assert.match(text, /homelab_server_disk_temperature_celsius\{server="elsa",device="\/dev\/sda"\} 33/); }); test("a disk on a steady climb has its days to full in /metrics, and a flat one does not", async () => { const { serverMetrics } = await import("../src/db/schema.js"); const day = 86_400_000; const rows = []; for (let i = 0; i < 10; i++) { const at = new Date(Date.now() - (10 - i) * day).toISOString(); rows.push({ serverId: id, metric: "disk:/data", value: 50 + i * 2, recordedAt: at }); rows.push({ serverId: id, metric: "disk:/", value: 30, recordedAt: at }); } await db.insert(serverMetrics).values(rows); const text = await buildMetrics(); const m = /homelab_server_disk_days_to_full\{server="elsa",mount="\/data"\} (\d+)/.exec(text); assert.ok(m, text.split("\n").filter((l) => l.includes("days_to_full")).join("\n")); assert.ok(Number(m[1]) >= 15 && Number(m[1]) <= 17, `about sixteen days (2 points a day, 32 left from 68%), got ${m[1]}`); assert.doesNotMatch(text, /days_to_full\{server="elsa",mount="\/"\}/); }); test("an agent that is now a newer version leaves a line in the server's history", async () => { await report({}, { agent_version: 3 }); await report({}, { agent_version: 4 }); const activity = await (await fetch(`${base}/servers/${id}/activity`)).json(); const line = activity.events.find((e: { title: string }) => /Agent updated to version 4/.test(e.title)); assert.ok(line, JSON.stringify(activity.events.map((e: { title: string }) => e.title))); assert.equal(line.detail, "from version 3"); assert.equal(activity.events.some((e: { title: string }) => /version 3$/.test(e.title)), false, "the first time a version is seen is not an update"); }); }); describe("agents updating themselves", () => { let app: Server; let base = ""; let who = "admin"; let id = 0; before(async () => { const [s] = await db.insert(servers).values({ name: "anna", apiTokenHash: hashToken("anna-tok"), apiTokenPrefix: "a", osType: "linux" }).returning(); id = s.id; const [w] = await db.insert(servers).values({ name: "win", apiTokenHash: hashToken("win-tok"), apiTokenPrefix: "w", osType: "windows", agentVersion: 1 }).returning(); void w; const a = express(); a.use("/api/agent/report", agentReportRouter); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/servers", serversRouter); app = await new Promise((resolve) => { const x = a.listen(0, "127.0.0.1", () => resolve(x)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`; }); after(() => app.close()); const ask = (token: string | null, query = "") => fetch(`${base}/agent/report/update${query}`, { headers: token ? { authorization: `Bearer ${token}` } : {} }); const put = (autoUpdate: unknown) => fetch(`${base}/servers/agent-update`, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ autoUpdate }) }); test("only an agent that proves who it is may ask", async () => { assert.equal((await ask(null)).status, 401); assert.equal((await ask("nobody")).status, 401); }); test("while the switch is off nobody is told to update, however far behind", async () => { const list = await (await fetch(`${base}/servers`)).json(); assert.equal(list.agentAutoUpdate, false, "off until an administrator turns it on"); const reply = await (await ask("anna-tok", "?version=1")).json(); assert.deepEqual(reply, { update: false, version: null, sha256: null }); }); test("only an administrator can switch it, it is written in the audit log, and it shows on the Servers list", async () => { who = "operator"; assert.equal((await put(true)).status, 403); who = "admin"; assert.equal((await put("yes")).status, 400); assert.equal((await put(true)).status, 200); assert.equal((await (await fetch(`${base}/servers`)).json()).agentAutoUpdate, true); const { auditLog } = await import("../src/db/schema.js"); const entry = (await db.select().from(auditLog)).find((e) => e.action === "agent_auto_update"); assert.ok(entry); assert.deepEqual(JSON.parse(entry.detail!), { from: false, to: true }); }); test("with it on, an agent that is behind is told to, with the checksum of what it will fetch; one that is current is not", async () => { const latest = latestAgentVersions().linux!; const behind = await (await ask("anna-tok", `?version=${latest - 1}`)).json(); assert.equal(behind.update, true); assert.equal(behind.version, latest); assert.equal(behind.sha256, latestAgentSha256("linux")); assert.match(behind.sha256, /^[0-9a-f]{64}$/); const current = await (await ask("anna-tok", `?version=${latest}`)).json(); assert.deepEqual(current, { update: false, version: null, sha256: null }); const ahead = await (await ask("anna-tok", `?version=${latest + 5}`)).json(); assert.equal(ahead.update, false); // no version given: what it last reported counts, and an agent that has said nothing is version 1 assert.equal((await (await ask("anna-tok")).json()).update, true); await db.update(servers).set({ agentVersion: latest }).where(eq(servers.id, id)); assert.equal((await (await ask("anna-tok")).json()).update, false); // nonsense is ignored in favour of that assert.equal((await (await ask("anna-tok", "?version=banana")).json()).update, false); }); test("each system is given its own script's checksum", async () => { const reply = await (await ask("win-tok", "?version=1")).json(); assert.equal(reply.update, true); assert.equal(reply.sha256, latestAgentSha256("windows")); assert.notEqual(reply.sha256, latestAgentSha256("linux")); }); test("the checksum is of the file that is served, byte for byte", async () => { const { createHash } = await import("node:crypto"); const { readFileSync } = await import("node:fs"); const { AGENT_DIR } = await import("../src/services/agentVersions.js"); const bytes = readFileSync(join(AGENT_DIR, "linux", "report-tasks.sh")); assert.equal(latestAgentSha256("linux"), createHash("sha256").update(bytes).digest("hex")); assert.equal(latestAgentSha256("linux", join(dir, "empty")), null, "an unreadable script has no checksum, so nobody is told to fetch it"); }); });