Files
Homelab-manager/server/test/hostHealthAndAgentUpdate.test.ts
T
bobbanandClaude Sonnet 5.5 06ac120a1d Services: opt-in monitoring with a per-server watch list
A failed service now raises an alert (and counts in /metrics and the public API) only when the server is told to watch it, by name or pattern, or * for all. Failed services that are not watched still show on the Host health card with a Watch button. Replaces the ignore list.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-16 21:48:32 +01:00

317 lines
17 KiB
TypeScript

/**
* What an agent can say about a machine's health (failed services, disk SMART, ZFS pools), the alerts and figures that come from it, and
* the switch that lets agents that are behind replace themselves.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-hosthealth-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, maintenanceWindows } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { hostHealthConditions, nextHostHealth, parseHostHealth } = await import("../src/services/hostHealth.js");
const { evaluateHealth } = await import("../src/services/healthMonitor.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { latestAgentSha256, latestAgentVersions } = await import("../src/services/agentVersions.js");
const { serversRouter } = await import("../src/routes/servers.js");
const { agentReportRouter } = await import("../src/routes/agentReport.js");
const { hashToken } = await import("../src/services/tokens.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const NOW = "2026-10-17T12:00:00.000Z";
const health = (extra: object = {}) => ({ reportedAt: NOW, ...extra });
describe("what a health report amounts to", () => {
test("failed services, a failing disk, a worn one and a pool that isn't online are each a problem; the healthy are not", () => {
const conditions = hostHealthConditions(7, "elsa", {
reportedAt: NOW,
failedUnits: [{ unit: "nginx.service", description: "" }, { unit: "backup.service", description: "" }],
smart: [
{ device: "/dev/sda", model: "WD Red", healthy: false, temperatureC: 40, wearPercent: null },
{ device: "/dev/nvme0", model: "Samsung", healthy: true, temperatureC: 38, wearPercent: 93 },
{ device: "/dev/sdb", model: "Fine", healthy: true, temperatureC: 31, wearPercent: 12 },
{ device: "/dev/sdc", model: "No verdict", healthy: null, temperatureC: null, wearPercent: null },
],
zfs: [{ name: "tank", health: "DEGRADED", capacityPercent: 40 }, { name: "rpool", health: "ONLINE", capacityPercent: 10 }],
});
const byKey = Object.fromEntries(conditions.map((c) => [c.key, c]));
assert.deepEqual(Object.keys(byKey).sort(), ["failed-units:server:7", "smart:server:7:/dev/nvme0", "smart:server:7:/dev/sda", "zfs:server:7:tank"]);
assert.equal(byKey["failed-units:server:7"].severity, "warning");
assert.match(byKey["failed-units:server:7"].message, /2 services have failed — nginx\.service, backup\.service/);
assert.equal(byKey["smart:server:7:/dev/sda"].severity, "critical");
assert.match(byKey["smart:server:7:/dev/sda"].message, /failing/);
assert.equal(byKey["smart:server:7:/dev/nvme0"].severity, "warning");
assert.match(byKey["smart:server:7:/dev/nvme0"].message, /93% of its rated life/);
assert.equal(byKey["zfs:server:7:tank"].severity, "critical");
assert.match(byKey["zfs:server:7:tank"].message, /DEGRADED/);
});
test("nothing said, or everything fine, is no problem; a long list of services is shortened", () => {
assert.deepEqual(hostHealthConditions(1, "x", null), []);
assert.deepEqual(hostHealthConditions(1, "x", health({ failedUnits: [], smart: [], zfs: [] })), []);
const many = Array.from({ length: 9 }, (_, i) => ({ unit: `u${i}.service`, description: "" }));
assert.match(hostHealthConditions(1, "x", health({ failedUnits: many }))[0].message, /and 5 more/);
assert.match(hostHealthConditions(1, "x", health({ failedUnits: [many[0]] }))[0].message, /1 service has failed/);
});
test("a report replaces the parts it carries and keeps the parts it doesn't", () => {
const first = nextHostHealth(null, { failed_units: [{ unit: "a.service" }], zfs: [{ name: "tank", health: "ONLINE", capacity_percent: 20 }] }, "t1");
assert.equal(parseHostHealth(first)!.failedUnits![0].unit, "a.service");
// the next report knows about disks only
const second = nextHostHealth(first, { smart: [{ device: "/dev/sda", healthy: true }] }, "t2");
const merged = parseHostHealth(second)!;
assert.equal(merged.failedUnits![0].unit, "a.service", "services kept");
assert.equal(merged.zfs![0].name, "tank", "pools kept");
assert.equal(merged.smart![0].device, "/dev/sda");
assert.equal(merged.reportedAt, "t2");
// an empty list is an answer ("none failed"), and clears the old one
assert.deepEqual(parseHostHealth(nextHostHealth(second, { failed_units: [] }, "t3"))!.failedUnits, []);
// nothing in the report: what was stored is untouched
assert.equal(nextHostHealth(second, {}, "t4"), second);
assert.equal(parseHostHealth("not json"), null);
});
test("an offline server isn't judged on its health, and an online one is", () => {
const snap = (lastSeenAt: string) => ({ servers: [{ id: 3, name: "elsa", lastSeenAt, disks: [], health: health({ failedUnits: [{ unit: "a.service", description: "" }] }) }], proxmox: [], synology: [] });
const now = Date.parse(NOW);
const limits = { serverOfflineMinutes: 60, diskUsagePercent: 90 };
const online = evaluateHealth(snap(new Date(now - 60_000).toISOString()), limits, now).map((c) => c.key);
assert.deepEqual(online, ["failed-units:server:3"]);
const offline = evaluateHealth(snap(new Date(now - 5 * 3600_000).toISOString()), limits, now).map((c) => c.key);
assert.deepEqual(offline, ["offline:server:3"]);
});
});
describe("through the report, the alerts, the figures and the page", () => {
let app: Server;
let base = "";
let id = 0;
let who = "admin";
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olle", role: "operator" },
]);
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", thresholds: JSON.stringify({ watchServices: ["*"] }) }).returning();
id = s.id;
const a = express();
a.use("/api/agent/report", agentReportRouter);
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/servers", serversRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const report = (system: unknown, extra: Record<string, unknown> = {}) =>
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", tasks: [], system, ...extra }) });
const detail = async () => (await (await fetch(`${base}/servers/${id}/detail`)).json()).server;
test("what an agent says is stored and shown; a garbled part costs the report nothing", async () => {
const res = await report({
health: {
failed_units: [{ unit: "nginx.service", description: "A high performance web server" }],
smart: [{ device: "/dev/sda", model: "WD Red", healthy: false, temperature_c: 41, wear_percent: null }],
zfs: "this isn't a list",
},
});
assert.equal(res.status, 202);
let s = await detail();
assert.equal(s.hostHealth.failedUnits[0].unit, "nginx.service");
assert.equal(s.hostHealth.smart[0].healthy, false);
assert.equal(s.hostHealth.zfs, undefined, "the part that made no sense was dropped");
// the list endpoint shows it too, and never as raw stored text
const row = (await (await fetch(`${base}/servers`)).json()).servers.find((x: { id: number }) => x.id === id);
assert.equal(typeof row.hostHealth, "object");
// a report with no health at all leaves it as it was; garbage in place of the whole thing too
assert.equal((await report({ cpu: { cores: 2 } })).status, 202);
assert.equal((await report({ health: 12 })).status, 202);
s = await detail();
assert.equal(s.hostHealth.failedUnits[0].unit, "nginx.service");
});
test("they appear on the Alerts page as their own kind, under the server, and a maintenance window silences them", async () => {
invalidateAlertsCache();
let alerts = (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`));
const services = alerts.find((a) => a.id.startsWith("services:failed-units"));
assert.ok(services, JSON.stringify(alerts.map((a) => a.id)));
assert.equal(services.category, "services");
assert.equal(services.link, `/servers/${id}?view=health`, "it opens the Health section of the server page");
assert.equal(services.silenced, false);
const disk = alerts.find((a) => a.id.startsWith("disk:smart"));
assert.ok(disk && disk.severity === "critical", "a failing disk is critical, in the disk category");
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: id, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
invalidateAlertsCache();
alerts = (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`));
assert.ok(alerts.length >= 2 && alerts.every((a) => a.silenced), "all held back under maintenance");
await db.delete(maintenanceWindows).where(eq(maintenanceWindows.targetId, id));
// the services come right once the agent says none have failed
await report({ health: { failed_units: [] } });
invalidateAlertsCache();
assert.equal((await getAlerts(true)).alerts.some((a) => a.id.startsWith("services:")), false);
});
test("/metrics carries them", async () => {
await report({ health: { failed_units: [{ unit: "a.service" }, { unit: "b.service" }], zfs: [{ name: "tank", health: "DEGRADED", capacity_percent: 12 }], smart: [{ device: "/dev/sda", healthy: true, temperature_c: 33, wear_percent: 5 }] } });
const text = await buildMetrics();
assert.match(text, /homelab_server_failed_units\{server="elsa"\} 2/);
assert.match(text, /homelab_server_zfs_pool_online\{server="elsa",pool="tank"\} 0/);
assert.match(text, /homelab_server_disk_healthy\{server="elsa",device="\/dev\/sda"\} 1/);
assert.match(text, /homelab_server_disk_wear_percent\{server="elsa",device="\/dev\/sda"\} 5/);
assert.match(text, /homelab_server_disk_temperature_celsius\{server="elsa",device="\/dev\/sda"\} 33/);
});
test("a disk on a steady climb has its days to full in /metrics, and a flat one does not", async () => {
const { serverMetrics } = await import("../src/db/schema.js");
const day = 86_400_000;
const rows = [];
for (let i = 0; i < 10; i++) {
const at = new Date(Date.now() - (10 - i) * day).toISOString();
rows.push({ serverId: id, metric: "disk:/data", value: 50 + i * 2, recordedAt: at });
rows.push({ serverId: id, metric: "disk:/", value: 30, recordedAt: at });
}
await db.insert(serverMetrics).values(rows);
const text = await buildMetrics();
const m = /homelab_server_disk_days_to_full\{server="elsa",mount="\/data"\} (\d+)/.exec(text);
assert.ok(m, text.split("\n").filter((l) => l.includes("days_to_full")).join("\n"));
assert.ok(Number(m[1]) >= 15 && Number(m[1]) <= 17, `about sixteen days (2 points a day, 32 left from 68%), got ${m[1]}`);
assert.doesNotMatch(text, /days_to_full\{server="elsa",mount="\/"\}/);
});
test("an agent that is now a newer version leaves a line in the server's history", async () => {
await report({}, { agent_version: 3 });
await report({}, { agent_version: 4 });
const activity = await (await fetch(`${base}/servers/${id}/activity`)).json();
const line = activity.events.find((e: { title: string }) => /Agent updated to version 4/.test(e.title));
assert.ok(line, JSON.stringify(activity.events.map((e: { title: string }) => e.title)));
assert.equal(line.detail, "from version 3");
assert.equal(activity.events.some((e: { title: string }) => /version 3$/.test(e.title)), false, "the first time a version is seen is not an update");
});
});
describe("agents updating themselves", () => {
let app: Server;
let base = "";
let who = "admin";
let id = 0;
before(async () => {
const [s] = await db.insert(servers).values({ name: "anna", apiTokenHash: hashToken("anna-tok"), apiTokenPrefix: "a", osType: "linux" }).returning();
id = s.id;
const [w] = await db.insert(servers).values({ name: "win", apiTokenHash: hashToken("win-tok"), apiTokenPrefix: "w", osType: "windows", agentVersion: 1 }).returning();
void w;
const a = express();
a.use("/api/agent/report", agentReportRouter);
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/servers", serversRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => app.close());
const ask = (token: string | null, query = "") => fetch(`${base}/agent/report/update${query}`, { headers: token ? { authorization: `Bearer ${token}` } : {} });
const put = (autoUpdate: unknown) => fetch(`${base}/servers/agent-update`, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ autoUpdate }) });
test("only an agent that proves who it is may ask", async () => {
assert.equal((await ask(null)).status, 401);
assert.equal((await ask("nobody")).status, 401);
});
test("while the switch is off nobody is told to update, however far behind", async () => {
const list = await (await fetch(`${base}/servers`)).json();
assert.equal(list.agentAutoUpdate, false, "off until an administrator turns it on");
const reply = await (await ask("anna-tok", "?version=1")).json();
assert.deepEqual(reply, { update: false, version: null, sha256: null });
});
test("only an administrator can switch it, it is written in the audit log, and it shows on the Servers list", async () => {
who = "operator";
assert.equal((await put(true)).status, 403);
who = "admin";
assert.equal((await put("yes")).status, 400);
assert.equal((await put(true)).status, 200);
assert.equal((await (await fetch(`${base}/servers`)).json()).agentAutoUpdate, true);
const { auditLog } = await import("../src/db/schema.js");
const entry = (await db.select().from(auditLog)).find((e) => e.action === "agent_auto_update");
assert.ok(entry);
assert.deepEqual(JSON.parse(entry.detail!), { from: false, to: true });
});
test("with it on, an agent that is behind is told to, with the checksum of what it will fetch; one that is current is not", async () => {
const latest = latestAgentVersions().linux!;
const behind = await (await ask("anna-tok", `?version=${latest - 1}`)).json();
assert.equal(behind.update, true);
assert.equal(behind.version, latest);
assert.equal(behind.sha256, latestAgentSha256("linux"));
assert.match(behind.sha256, /^[0-9a-f]{64}$/);
const current = await (await ask("anna-tok", `?version=${latest}`)).json();
assert.deepEqual(current, { update: false, version: null, sha256: null });
const ahead = await (await ask("anna-tok", `?version=${latest + 5}`)).json();
assert.equal(ahead.update, false);
// no version given: what it last reported counts, and an agent that has said nothing is version 1
assert.equal((await (await ask("anna-tok")).json()).update, true);
await db.update(servers).set({ agentVersion: latest }).where(eq(servers.id, id));
assert.equal((await (await ask("anna-tok")).json()).update, false);
// nonsense is ignored in favour of that
assert.equal((await (await ask("anna-tok", "?version=banana")).json()).update, false);
});
test("each system is given its own script's checksum", async () => {
const reply = await (await ask("win-tok", "?version=1")).json();
assert.equal(reply.update, true);
assert.equal(reply.sha256, latestAgentSha256("windows"));
assert.notEqual(reply.sha256, latestAgentSha256("linux"));
});
test("the checksum is of the file that is served, byte for byte", async () => {
const { createHash } = await import("node:crypto");
const { readFileSync } = await import("node:fs");
const { AGENT_DIR } = await import("../src/services/agentVersions.js");
const bytes = readFileSync(join(AGENT_DIR, "linux", "report-tasks.sh"));
assert.equal(latestAgentSha256("linux"), createHash("sha256").update(bytes).digest("hex"));
assert.equal(latestAgentSha256("linux", join(dir, "empty")), null, "an unreadable script has no checksum, so nobody is told to fetch it");
});
});