Files
Homelab-manager/server/test/agentDiagnostics.test.ts
bobbanandClaude Sonnet 5.5 06ac120a1d Services: opt-in monitoring with a per-server watch list
A failed service now raises an alert (and counts in /metrics and the public API) only when the server is told to watch it, by name or pattern, or * for all. Failed services that are not watched still show on the Host health card with a Watch button. Replaces the ignore list.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-16 21:48:32 +01:00

217 lines
13 KiB
TypeScript

/**
* What an agent could not collect and what to do about it, and the services a server is told to ignore.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-diag-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { describeDiagnostics, parseDiagnostics } = await import("../src/services/agentDiagnostics.js");
const { isServiceWatched, onlyWatchedServices, markWatchedServices } = await import("../src/services/hostHealth.js");
const { checkThresholds, parseThresholds } = await import("../src/services/serverThresholds.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { serversRouter } = await import("../src/routes/servers.js");
const { agentReportRouter } = await import("../src/routes/agentReport.js");
const { hashToken } = await import("../src/services/tokens.js");
await runMigrations();
const latest = { linux: 6, windows: 5 };
const ctx = (extra: Partial<{ osType: string; lastSeenAt: string | null; agentVersion: number | null }> = {}) => ({ osType: "linux", lastSeenAt: "2026-10-18T10:00:00Z", agentVersion: 6, latest, ...extra });
const stored = (items: unknown[]) => JSON.stringify({ items, reportedAt: "2026-10-18T10:00:00Z" });
describe("saying what to do about what couldn't be collected", () => {
test("a missing tool is a tip with the fix; ZFS that isn't there is not worth saying", () => {
const notes = describeDiagnostics(
parseDiagnostics(stored([
{ check: "smart", status: "unavailable", reason: "tool_missing", detail: "smartctl" },
{ check: "zfs", status: "unavailable", reason: "tool_missing", detail: "zpool" },
{ check: "hardware", status: "unavailable", reason: "not_found" },
])),
ctx(),
);
assert.deepEqual(notes.map((n) => n.check), ["smart", "hardware"], "no word about ZFS");
assert.equal(notes[0].level, "info");
assert.match(notes[0].fix!, /smartmontools/);
});
test("a failure is a warning that says what the agent said, with a fix for the system it is on", () => {
const failed = stored([{ check: "updates", status: "failed", reason: "error", detail: "apt did not answer" }]);
const linux = describeDiagnostics(parseDiagnostics(failed), ctx())[0];
assert.equal(linux.level, "warning");
assert.equal(linux.detail, "apt did not answer");
assert.match(linux.fix!, /--check-updates/);
const windows = describeDiagnostics(parseDiagnostics(failed), ctx({ osType: "windows", agentVersion: 5 }))[0];
assert.match(windows.fix!, /Windows Update/);
});
test("a step that didn't run is a warning, and warnings come first", () => {
const notes = describeDiagnostics(parseDiagnostics(stored([{ check: "hardware", status: "unavailable", reason: "not_found" }, { check: "prepare", status: "unavailable", reason: "step_missing" }])), ctx());
assert.deepEqual(notes.map((n) => [n.check, n.level]), [["prepare", "warning"], ["hardware", "info"]]);
});
test("something it has no words for is still said, rather than dropped", () => {
const notes = describeDiagnostics(parseDiagnostics(stored([{ check: "gpu_temps", status: "failed", reason: "error" }, { check: "fans", status: "unavailable", reason: "no_sensor" }])), ctx());
assert.equal(notes[0].level, "warning");
assert.match(notes[0].title, /gpu_temps/);
assert.match(notes[1].title, /fans isn't available \(no sensor\)/);
});
test("an agent that is behind is told so, whether or not it said anything", () => {
const notes = describeDiagnostics(null, ctx({ agentVersion: 3 }));
assert.equal(notes.length, 1);
assert.match(notes[0].title, /version 3; 6 is current/);
assert.deepEqual(describeDiagnostics(null, ctx()), [], "an agent that is current and silent has nothing to say");
assert.deepEqual(describeDiagnostics(null, ctx({ lastSeenAt: null, agentVersion: 3 })), [], "a server that has never reported has no agent to update");
});
test("what is stored is read leniently", () => {
assert.equal(parseDiagnostics(null), null);
assert.equal(parseDiagnostics("not json"), null);
const read = parseDiagnostics(stored([{ check: "ok", status: "fine" }, "x", { check: "smart", status: "failed", detail: "a\nb" }]))!;
assert.deepEqual(read.items, [{ check: "smart", status: "failed", detail: "a b" }]);
});
});
describe("services a server is told to watch", () => {
test("names and patterns, in any case, with or without .service", () => {
assert.equal(isServiceWatched("gupdate.service", ["gupdate"]), true);
assert.equal(isServiceWatched("GUpdate", ["gupdate*"]), true);
assert.equal(isServiceWatched("snap.lxd.activate.service", ["snap.*"]), true);
assert.equal(isServiceWatched("nginx.service", ["gupdate*", "snap.*"]), false);
assert.equal(isServiceWatched("nginx.service", ["nginx.service"]), true);
assert.equal(isServiceWatched("mynginx.service", ["nginx"]), false, "a name is the whole name");
assert.equal(isServiceWatched("a+b.service", ["a+b"]), true, "characters that mean something to a regex are only characters here");
assert.equal(isServiceWatched("anything", []), false);
assert.equal(isServiceWatched("anything", undefined), false);
});
test("* is every service", () => {
assert.equal(isServiceWatched("anything.service", ["*"]), true);
});
test("only the watched are kept for what is judged, all are shown marked", () => {
const health = { failedUnits: [{ unit: "nginx.service", description: "" }, { unit: "gupdate", description: "Google Update" }], reportedAt: "t" };
assert.deepEqual(onlyWatchedServices(health, ["nginx"])!.failedUnits!.map((u) => u.unit), ["nginx.service"]);
assert.deepEqual(markWatchedServices(health, ["nginx"])!.failedUnits!.map((u) => [u.unit, u.watched]), [["nginx.service", true], ["gupdate", false]]);
assert.equal(onlyWatchedServices(null, ["x"]), null);
assert.deepEqual(onlyWatchedServices(health, [])!.failedUnits, [], "nothing is watched until it is said: no service counts as failed");
assert.deepEqual(onlyWatchedServices(health, undefined)!.failedUnits, []);
assert.deepEqual(markWatchedServices(health, [])!.failedUnits!.map((u) => u.watched), [false, false]);
const silent = { reportedAt: "t" };
assert.deepEqual(onlyWatchedServices(silent, []), silent, "a report with nothing about services is left as it is");
});
test("the list is part of a server's own thresholds: kept tidy, and a wrong one is refused", () => {
const ok = checkThresholds({ watchServices: [" gupdate* ", "gupdate*", "", "snap.*"] });
assert.deepEqual(ok.ok && ok.value, { watchServices: ["gupdate*", "snap.*"] });
assert.equal(checkThresholds({ watchServices: "gupdate" }).ok, false);
assert.equal(checkThresholds({ watchServices: [3] }).ok, false);
assert.equal(checkThresholds({ watchServices: ["x".repeat(101)] }).ok, false);
assert.equal(checkThresholds({ watchServices: Array.from({ length: 51 }, (_, i) => `s${i}`) }).ok, false);
assert.equal(checkThresholds({ watchServices: [] }).ok && (checkThresholds({ watchServices: [] }) as { value: unknown }).value, null, "an empty list is no thresholds at all");
assert.deepEqual(parseThresholds(JSON.stringify({ watchServices: ["a", 3, "", "a"] })), { watchServices: ["a"] });
});
});
describe("through a report, the page, the alerts and the figures", () => {
let app: Server;
let base = "";
let id = 0;
before(async () => {
await db.insert(users).values([{ oidcSub: "admin", name: "Ada", role: "admin" }]);
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: new Date().toISOString(), agentVersion: 8 }).returning();
id = s.id;
const a = express();
a.use("/api/agent/report", agentReportRouter);
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
next();
});
a.use("/api/servers", serversRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const report = (system: unknown) =>
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", agent_version: 8, tasks: [], system }) });
const detail = async () => await (await fetch(`${base}/servers/${id}/detail`)).json();
const patch = (body: unknown) => fetch(`${base}/servers/${id}`, { method: "PATCH", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
test("what the agent says it couldn't collect is shown with what to do; garbage entries are dropped; silence keeps what was said", async () => {
assert.equal(
(await report({ diagnostics: [{ check: "smart", status: "unavailable", reason: "tool_missing", detail: "smartctl" }, { check: "x", status: "bogus" }, "junk", { status: "failed" }] })).status,
202,
);
let d = (await detail()).diagnostics;
assert.deepEqual(d.map((n: { check: string }) => n.check), ["smart"]);
assert.match(d[0].fix, /smartmontools/);
assert.equal((await report({ cpu: { cores: 2 } })).status, 202);
assert.equal((await detail()).diagnostics.length, 1, "a report that doesn't say keeps what was said");
assert.equal((await report({ diagnostics: "garbage" })).status, 202);
assert.equal((await detail()).diagnostics.length, 1, "and so does one that isn't a list");
assert.equal((await report({ diagnostics: [] })).status, 202);
assert.deepEqual((await detail()).diagnostics, [], "an empty list says nothing was missed");
const row = (await (await fetch(`${base}/servers`)).json()).servers.find((x: { id: number }) => x.id === id);
assert.equal(row.agentDiagnostics, undefined, "never as raw stored text");
});
test("nothing is an alert until it is watched; a watched service that has failed is, and a failed one that is not watched still shows on the page", async () => {
assert.equal((await report({ health: { failed_units: [{ unit: "nginx.service", description: "web" }, { unit: "gupdate.service", description: "Google Update" }] } })).status, 202);
invalidateAlertsCache();
const failing = async () => (await getAlerts(true)).alerts.find((a) => a.id.startsWith("services:failed-units") && a.id.includes(`:server:${id}`));
assert.equal(await failing(), undefined, "nothing is watched, so nothing is an alert");
assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\{server="elsa"[^}]*\} 0`));
assert.deepEqual((await detail()).server.hostHealth.failedUnits.map((u: { unit: string; watched?: boolean }) => [u.unit, u.watched]), [["nginx.service", false], ["gupdate.service", false]], "still shown, marked not watched");
assert.equal((await patch({ thresholds: { watchServices: ["nginx"] } })).status, 200);
assert.match((await failing())!.message, /1 service has failed — nginx\.service/, "the alert cache was dropped when the list changed");
assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\{server="elsa"[^}]*\} 1`));
const units = (await detail()).server.hostHealth.failedUnits;
assert.deepEqual(units.map((u: { unit: string; watched?: boolean }) => [u.unit, u.watched]), [["nginx.service", true], ["gupdate.service", false]]);
assert.equal((await patch({ thresholds: { watchServices: ["*"] } })).status, 200);
assert.match((await failing())!.message, /2 services have failed/, "* watches them all, as it always was");
assert.equal((await patch({ thresholds: null })).status, 200);
assert.equal(await failing(), undefined, "taking the list away watches nothing again");
});
test("a list that isn't one is refused with a message", async () => {
const res = await patch({ thresholds: { watchServices: "gupdate" } });
assert.equal(res.status, 400);
assert.match((await res.json()).message, /should be a list/);
});
});