Agent diagnostics ("what the agent couldn't collect") and services to ignore per server

- Agents (Linux 6, Windows 5) note each check they couldn't make and why; the server page turns that into tips and fixes, including when the agent is behind
- Linux --prepare leaves its notes for the report; a missing prepare step is itself noted
- Services to ignore are set per server from the Host health card (names or * patterns), applied on the server so alerts and /metrics drop them at once; saving alert limits keeps the list

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 22:16:31 +02:00
1 parent b5384c1e03
commit c3bc6bbf72
26 files changed
+4118 -45

No files matched your search

+207
View File
@@ -0,0 +1,207 @@
/**
* What an agent could not collect and what to do about it, and the services a server is told to ignore.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-diag-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { describeDiagnostics, parseDiagnostics } = await import("../src/services/agentDiagnostics.js");
const { isServiceIgnored, withoutIgnoredServices, markIgnoredServices } = await import("../src/services/hostHealth.js");
const { checkThresholds, parseThresholds } = await import("../src/services/serverThresholds.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { serversRouter } = await import("../src/routes/servers.js");
const { agentReportRouter } = await import("../src/routes/agentReport.js");
const { hashToken } = await import("../src/services/tokens.js");
await runMigrations();
const latest = { linux: 6, windows: 5 };
const ctx = (extra: Partial<{ osType: string; lastSeenAt: string | null; agentVersion: number | null }> = {}) => ({ osType: "linux", lastSeenAt: "2026-10-18T10:00:00Z", agentVersion: 6, latest, ...extra });
const stored = (items: unknown[]) => JSON.stringify({ items, reportedAt: "2026-10-18T10:00:00Z" });
describe("saying what to do about what couldn't be collected", () => {
test("a missing tool is a tip with the fix; ZFS that isn't there is not worth saying", () => {
const notes = describeDiagnostics(
parseDiagnostics(stored([
{ check: "smart", status: "unavailable", reason: "tool_missing", detail: "smartctl" },
{ check: "zfs", status: "unavailable", reason: "tool_missing", detail: "zpool" },
{ check: "hardware", status: "unavailable", reason: "not_found" },
])),
ctx(),
);
assert.deepEqual(notes.map((n) => n.check), ["smart", "hardware"], "no word about ZFS");
assert.equal(notes[0].level, "info");
assert.match(notes[0].fix!, /smartmontools/);
});
test("a failure is a warning that says what the agent said, with a fix for the system it is on", () => {
const failed = stored([{ check: "updates", status: "failed", reason: "error", detail: "apt did not answer" }]);
const linux = describeDiagnostics(parseDiagnostics(failed), ctx())[0];
assert.equal(linux.level, "warning");
assert.equal(linux.detail, "apt did not answer");
assert.match(linux.fix!, /--check-updates/);
const windows = describeDiagnostics(parseDiagnostics(failed), ctx({ osType: "windows", agentVersion: 5 }))[0];
assert.match(windows.fix!, /Windows Update/);
});
test("a step that didn't run is a warning, and warnings come first", () => {
const notes = describeDiagnostics(parseDiagnostics(stored([{ check: "hardware", status: "unavailable", reason: "not_found" }, { check: "prepare", status: "unavailable", reason: "step_missing" }])), ctx());
assert.deepEqual(notes.map((n) => [n.check, n.level]), [["prepare", "warning"], ["hardware", "info"]]);
});
test("something it has no words for is still said, rather than dropped", () => {
const notes = describeDiagnostics(parseDiagnostics(stored([{ check: "gpu_temps", status: "failed", reason: "error" }, { check: "fans", status: "unavailable", reason: "no_sensor" }])), ctx());
assert.equal(notes[0].level, "warning");
assert.match(notes[0].title, /gpu_temps/);
assert.match(notes[1].title, /fans isn't available \(no sensor\)/);
});
test("an agent that is behind is told so, whether or not it said anything", () => {
const notes = describeDiagnostics(null, ctx({ agentVersion: 3 }));
assert.equal(notes.length, 1);
assert.match(notes[0].title, /version 3; 6 is current/);
assert.deepEqual(describeDiagnostics(null, ctx()), [], "an agent that is current and silent has nothing to say");
assert.deepEqual(describeDiagnostics(null, ctx({ lastSeenAt: null, agentVersion: 3 })), [], "a server that has never reported has no agent to update");
});
test("what is stored is read leniently", () => {
assert.equal(parseDiagnostics(null), null);
assert.equal(parseDiagnostics("not json"), null);
const read = parseDiagnostics(stored([{ check: "ok", status: "fine" }, "x", { check: "smart", status: "failed", detail: "a\nb" }]))!;
assert.deepEqual(read.items, [{ check: "smart", status: "failed", detail: "a b" }]);
});
});
describe("services a server is told to ignore", () => {
test("names and patterns, in any case, with or without .service", () => {
assert.equal(isServiceIgnored("gupdate.service", ["gupdate"]), true);
assert.equal(isServiceIgnored("GUpdate", ["gupdate*"]), true);
assert.equal(isServiceIgnored("snap.lxd.activate.service", ["snap.*"]), true);
assert.equal(isServiceIgnored("nginx.service", ["gupdate*", "snap.*"]), false);
assert.equal(isServiceIgnored("nginx.service", ["nginx.service"]), true);
assert.equal(isServiceIgnored("mynginx.service", ["nginx"]), false, "a name is the whole name");
assert.equal(isServiceIgnored("a+b.service", ["a+b"]), true, "characters that mean something to a regex are only characters here");
assert.equal(isServiceIgnored("anything", []), false);
assert.equal(isServiceIgnored("anything", undefined), false);
});
test("left out for what is judged, marked for what is shown", () => {
const health = { failedUnits: [{ unit: "nginx.service", description: "" }, { unit: "gupdate", description: "Google Update" }], reportedAt: "t" };
assert.deepEqual(withoutIgnoredServices(health, ["gupdate*"])!.failedUnits!.map((u) => u.unit), ["nginx.service"]);
assert.deepEqual(markIgnoredServices(health, ["gupdate*"])!.failedUnits!.map((u) => [u.unit, u.ignored ?? false]), [["nginx.service", false], ["gupdate", true]]);
assert.equal(withoutIgnoredServices(null, ["x"]), null);
assert.deepEqual(withoutIgnoredServices(health, []), health);
});
test("the list is part of a server's own thresholds: kept tidy, and a wrong one is refused", () => {
const ok = checkThresholds({ ignoreServices: [" gupdate* ", "gupdate*", "", "snap.*"] });
assert.deepEqual(ok.ok && ok.value, { ignoreServices: ["gupdate*", "snap.*"] });
assert.equal(checkThresholds({ ignoreServices: "gupdate" }).ok, false);
assert.equal(checkThresholds({ ignoreServices: [3] }).ok, false);
assert.equal(checkThresholds({ ignoreServices: ["x".repeat(101)] }).ok, false);
assert.equal(checkThresholds({ ignoreServices: Array.from({ length: 31 }, (_, i) => `s${i}`) }).ok, false);
assert.equal(checkThresholds({ ignoreServices: [] }).ok && (checkThresholds({ ignoreServices: [] }) as { value: unknown }).value, null, "an empty list is no thresholds at all");
assert.deepEqual(parseThresholds(JSON.stringify({ ignoreServices: ["a", 3, "", "a"] })), { ignoreServices: ["a"] });
});
});
describe("through a report, the page, the alerts and the figures", () => {
let app: Server;
let base = "";
let id = 0;
before(async () => {
await db.insert(users).values([{ oidcSub: "admin", name: "Ada", role: "admin" }]);
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: new Date().toISOString(), agentVersion: 6 }).returning();
id = s.id;
const a = express();
a.use("/api/agent/report", agentReportRouter);
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
next();
});
a.use("/api/servers", serversRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const report = (system: unknown) =>
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", agent_version: 6, tasks: [], system }) });
const detail = async () => await (await fetch(`${base}/servers/${id}/detail`)).json();
const patch = (body: unknown) => fetch(`${base}/servers/${id}`, { method: "PATCH", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
test("what the agent says it couldn't collect is shown with what to do; garbage entries are dropped; silence keeps what was said", async () => {
assert.equal(
(await report({ diagnostics: [{ check: "smart", status: "unavailable", reason: "tool_missing", detail: "smartctl" }, { check: "x", status: "bogus" }, "junk", { status: "failed" }] })).status,
202,
);
let d = (await detail()).diagnostics;
assert.deepEqual(d.map((n: { check: string }) => n.check), ["smart"]);
assert.match(d[0].fix, /smartmontools/);
assert.equal((await report({ cpu: { cores: 2 } })).status, 202);
assert.equal((await detail()).diagnostics.length, 1, "a report that doesn't say keeps what was said");
assert.equal((await report({ diagnostics: "garbage" })).status, 202);
assert.equal((await detail()).diagnostics.length, 1, "and so does one that isn't a list");
assert.equal((await report({ diagnostics: [] })).status, 202);
assert.deepEqual((await detail()).diagnostics, [], "an empty list says nothing was missed");
const row = (await (await fetch(`${base}/servers`)).json()).servers.find((x: { id: number }) => x.id === id);
assert.equal(row.agentDiagnostics, undefined, "never as raw stored text");
});
test("a service the server is told to ignore stops being an alert and a figure, still shows on the page marked, and can be un-ignored", async () => {
assert.equal((await report({ health: { failed_units: [{ unit: "nginx.service", description: "web" }, { unit: "gupdate.service", description: "Google Update" }] } })).status, 202);
invalidateAlertsCache();
const failing = async () => (await getAlerts(true)).alerts.find((a) => a.id.startsWith("services:failed-units") && a.id.includes(`:server:${id}`));
assert.match((await failing())!.message, /2 services have failed/);
assert.equal((await patch({ thresholds: { ignoreServices: ["gupdate*"] } })).status, 200);
assert.match((await failing())!.message, /1 service has failed — nginx\.service/, "the alert cache was dropped when the list changed");
assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\\{server="elsa"[^}]*\\} 1`));
const units = (await detail()).server.hostHealth.failedUnits;
assert.deepEqual(units.map((u: { unit: string; ignored?: boolean }) => [u.unit, u.ignored ?? false]), [["nginx.service", false], ["gupdate.service", true]]);
assert.equal((await patch({ thresholds: { ignoreServices: ["gupdate*", "nginx"] } })).status, 200);
assert.equal(await failing(), undefined, "with every failed service ignored there is nothing to alert about");
assert.equal((await patch({ thresholds: null })).status, 200);
assert.match((await failing())!.message, /2 services have failed/);
});
test("a list that isn't one is refused with a message", async () => {
const res = await patch({ thresholds: { ignoreServices: "gupdate" } });
assert.equal(res.status, 400);
assert.match((await res.json()).message, /should be a list/);
});
});
+18 -2
View File
@@ -32,9 +32,9 @@ function run(snippet: string): unknown {
}
describe("the Windows agent's health parts", { skip: !havePwsh }, () => {
test("it says it is version 4, and stays plain ASCII", () => {
test("it says it is version 5, and stays plain ASCII", () => {
const text = readFileSync(script, "utf8");
assert.match(text, /^\$script:AgentVersion = 4\s*$/m);
assert.match(text, /^\$script:AgentVersion = 5\s*$/m);
assert.ok(!/[^\x00-\x7f]/.test(text));
});
@@ -103,6 +103,22 @@ describe("the Windows agent's health parts", { skip: !havePwsh }, () => {
assert.deepEqual(out.map((p) => [p.name, p.health, p.capacity_percent]), [["Media", "ONLINE", 25], ["Backups", "DEGRADED", 90], ["Broken", "FAULTED", null]]);
});
test("what could not be read is noted, with a word for why and at most a line of what Windows said", () => {
const out = run(`
function Get-CimInstance { throw "WMI is unhappy\`r\`nsecond line" }
$WarningPreference = 'SilentlyContinue'
$null = Get-HostHealth @{ IgnoreServices = @() }
Add-Diag 'hardware' 'unavailable' 'not_found'
Add-Diag 'updates' 'failed' 'error' ('x' * 400)
ConvertTo-Json -InputObject @{ items = @($script:Diag) } -Depth 4 -Compress
`) as { items: { check: string; status: string; reason?: string; detail?: string }[] };
const byCheck = Object.fromEntries(out.items.map((i) => [i.check, i]));
assert.deepEqual([byCheck.failed_units.status, byCheck.failed_units.reason], ["failed", "error"]);
assert.equal(byCheck.failed_units.detail, "WMI is unhappy second line", "line breaks are flattened");
assert.deepEqual(byCheck.hardware, { check: "hardware", status: "unavailable", reason: "not_found" }, "nothing to say about it, no detail");
assert.equal(byCheck.updates.detail!.length, 160, "cut to a line");
});
test("a maker's placeholders aren't a model or a serial number", () => {
const out = run(`
$t = @('To Be Filled By O.E.M.', 'Default string', 'System Serial Number', 'System Product Name', '0000000', ' ', 'None', 'Unknown', 'ASUS', 'ROG STRIX B560-F', 'PF2ABCDE', '0A1B')