Linux agent 8 and Windows agent 7 say how long the machine has been up; the server stores it (migration 0086) and shows it as Up for in the status strip (left out while offline) and Uptime on the server page. The Integrations header wraps and its button no longer squeezes, and .table-responsive is positioned so hidden text in scrolled-off columns cannot widen the page. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
208 lines
12 KiB
TypeScript
208 lines
12 KiB
TypeScript
/**
|
|
* What an agent could not collect and what to do about it, and the services a server is told to ignore.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { AddressInfo } from "node:net";
|
|
import type { Server } from "node:http";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-diag-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const express = (await import("express")).default;
|
|
const { db } = await import("../src/db/client.js");
|
|
const { users, servers } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const { describeDiagnostics, parseDiagnostics } = await import("../src/services/agentDiagnostics.js");
|
|
const { isServiceIgnored, withoutIgnoredServices, markIgnoredServices } = await import("../src/services/hostHealth.js");
|
|
const { checkThresholds, parseThresholds } = await import("../src/services/serverThresholds.js");
|
|
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
|
const { buildMetrics } = await import("../src/services/metrics.js");
|
|
const { serversRouter } = await import("../src/routes/servers.js");
|
|
const { agentReportRouter } = await import("../src/routes/agentReport.js");
|
|
const { hashToken } = await import("../src/services/tokens.js");
|
|
|
|
await runMigrations();
|
|
|
|
const latest = { linux: 6, windows: 5 };
|
|
const ctx = (extra: Partial<{ osType: string; lastSeenAt: string | null; agentVersion: number | null }> = {}) => ({ osType: "linux", lastSeenAt: "2026-10-18T10:00:00Z", agentVersion: 6, latest, ...extra });
|
|
const stored = (items: unknown[]) => JSON.stringify({ items, reportedAt: "2026-10-18T10:00:00Z" });
|
|
|
|
describe("saying what to do about what couldn't be collected", () => {
|
|
test("a missing tool is a tip with the fix; ZFS that isn't there is not worth saying", () => {
|
|
const notes = describeDiagnostics(
|
|
parseDiagnostics(stored([
|
|
{ check: "smart", status: "unavailable", reason: "tool_missing", detail: "smartctl" },
|
|
{ check: "zfs", status: "unavailable", reason: "tool_missing", detail: "zpool" },
|
|
{ check: "hardware", status: "unavailable", reason: "not_found" },
|
|
])),
|
|
ctx(),
|
|
);
|
|
assert.deepEqual(notes.map((n) => n.check), ["smart", "hardware"], "no word about ZFS");
|
|
assert.equal(notes[0].level, "info");
|
|
assert.match(notes[0].fix!, /smartmontools/);
|
|
});
|
|
|
|
test("a failure is a warning that says what the agent said, with a fix for the system it is on", () => {
|
|
const failed = stored([{ check: "updates", status: "failed", reason: "error", detail: "apt did not answer" }]);
|
|
const linux = describeDiagnostics(parseDiagnostics(failed), ctx())[0];
|
|
assert.equal(linux.level, "warning");
|
|
assert.equal(linux.detail, "apt did not answer");
|
|
assert.match(linux.fix!, /--check-updates/);
|
|
const windows = describeDiagnostics(parseDiagnostics(failed), ctx({ osType: "windows", agentVersion: 5 }))[0];
|
|
assert.match(windows.fix!, /Windows Update/);
|
|
});
|
|
|
|
test("a step that didn't run is a warning, and warnings come first", () => {
|
|
const notes = describeDiagnostics(parseDiagnostics(stored([{ check: "hardware", status: "unavailable", reason: "not_found" }, { check: "prepare", status: "unavailable", reason: "step_missing" }])), ctx());
|
|
assert.deepEqual(notes.map((n) => [n.check, n.level]), [["prepare", "warning"], ["hardware", "info"]]);
|
|
});
|
|
|
|
test("something it has no words for is still said, rather than dropped", () => {
|
|
const notes = describeDiagnostics(parseDiagnostics(stored([{ check: "gpu_temps", status: "failed", reason: "error" }, { check: "fans", status: "unavailable", reason: "no_sensor" }])), ctx());
|
|
assert.equal(notes[0].level, "warning");
|
|
assert.match(notes[0].title, /gpu_temps/);
|
|
assert.match(notes[1].title, /fans isn't available \(no sensor\)/);
|
|
});
|
|
|
|
test("an agent that is behind is told so, whether or not it said anything", () => {
|
|
const notes = describeDiagnostics(null, ctx({ agentVersion: 3 }));
|
|
assert.equal(notes.length, 1);
|
|
assert.match(notes[0].title, /version 3; 6 is current/);
|
|
assert.deepEqual(describeDiagnostics(null, ctx()), [], "an agent that is current and silent has nothing to say");
|
|
assert.deepEqual(describeDiagnostics(null, ctx({ lastSeenAt: null, agentVersion: 3 })), [], "a server that has never reported has no agent to update");
|
|
});
|
|
|
|
test("what is stored is read leniently", () => {
|
|
assert.equal(parseDiagnostics(null), null);
|
|
assert.equal(parseDiagnostics("not json"), null);
|
|
const read = parseDiagnostics(stored([{ check: "ok", status: "fine" }, "x", { check: "smart", status: "failed", detail: "a\nb" }]))!;
|
|
assert.deepEqual(read.items, [{ check: "smart", status: "failed", detail: "a b" }]);
|
|
});
|
|
});
|
|
|
|
describe("services a server is told to ignore", () => {
|
|
test("names and patterns, in any case, with or without .service", () => {
|
|
assert.equal(isServiceIgnored("gupdate.service", ["gupdate"]), true);
|
|
assert.equal(isServiceIgnored("GUpdate", ["gupdate*"]), true);
|
|
assert.equal(isServiceIgnored("snap.lxd.activate.service", ["snap.*"]), true);
|
|
assert.equal(isServiceIgnored("nginx.service", ["gupdate*", "snap.*"]), false);
|
|
assert.equal(isServiceIgnored("nginx.service", ["nginx.service"]), true);
|
|
assert.equal(isServiceIgnored("mynginx.service", ["nginx"]), false, "a name is the whole name");
|
|
assert.equal(isServiceIgnored("a+b.service", ["a+b"]), true, "characters that mean something to a regex are only characters here");
|
|
assert.equal(isServiceIgnored("anything", []), false);
|
|
assert.equal(isServiceIgnored("anything", undefined), false);
|
|
});
|
|
|
|
test("left out for what is judged, marked for what is shown", () => {
|
|
const health = { failedUnits: [{ unit: "nginx.service", description: "" }, { unit: "gupdate", description: "Google Update" }], reportedAt: "t" };
|
|
assert.deepEqual(withoutIgnoredServices(health, ["gupdate*"])!.failedUnits!.map((u) => u.unit), ["nginx.service"]);
|
|
assert.deepEqual(markIgnoredServices(health, ["gupdate*"])!.failedUnits!.map((u) => [u.unit, u.ignored ?? false]), [["nginx.service", false], ["gupdate", true]]);
|
|
assert.equal(withoutIgnoredServices(null, ["x"]), null);
|
|
assert.deepEqual(withoutIgnoredServices(health, []), health);
|
|
});
|
|
|
|
test("the list is part of a server's own thresholds: kept tidy, and a wrong one is refused", () => {
|
|
const ok = checkThresholds({ ignoreServices: [" gupdate* ", "gupdate*", "", "snap.*"] });
|
|
assert.deepEqual(ok.ok && ok.value, { ignoreServices: ["gupdate*", "snap.*"] });
|
|
assert.equal(checkThresholds({ ignoreServices: "gupdate" }).ok, false);
|
|
assert.equal(checkThresholds({ ignoreServices: [3] }).ok, false);
|
|
assert.equal(checkThresholds({ ignoreServices: ["x".repeat(101)] }).ok, false);
|
|
assert.equal(checkThresholds({ ignoreServices: Array.from({ length: 31 }, (_, i) => `s${i}`) }).ok, false);
|
|
assert.equal(checkThresholds({ ignoreServices: [] }).ok && (checkThresholds({ ignoreServices: [] }) as { value: unknown }).value, null, "an empty list is no thresholds at all");
|
|
assert.deepEqual(parseThresholds(JSON.stringify({ ignoreServices: ["a", 3, "", "a"] })), { ignoreServices: ["a"] });
|
|
});
|
|
});
|
|
|
|
describe("through a report, the page, the alerts and the figures", () => {
|
|
let app: Server;
|
|
let base = "";
|
|
let id = 0;
|
|
|
|
before(async () => {
|
|
await db.insert(users).values([{ oidcSub: "admin", name: "Ada", role: "admin" }]);
|
|
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: new Date().toISOString(), agentVersion: 8 }).returning();
|
|
id = s.id;
|
|
const a = express();
|
|
a.use("/api/agent/report", agentReportRouter);
|
|
a.use(express.json());
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
|
|
next();
|
|
});
|
|
a.use("/api/servers", serversRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
|
});
|
|
|
|
after(() => {
|
|
app.close();
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows keeps the database file locked until the process exits.
|
|
}
|
|
});
|
|
|
|
const report = (system: unknown) =>
|
|
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", agent_version: 8, tasks: [], system }) });
|
|
const detail = async () => await (await fetch(`${base}/servers/${id}/detail`)).json();
|
|
const patch = (body: unknown) => fetch(`${base}/servers/${id}`, { method: "PATCH", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
|
|
|
|
test("what the agent says it couldn't collect is shown with what to do; garbage entries are dropped; silence keeps what was said", async () => {
|
|
assert.equal(
|
|
(await report({ diagnostics: [{ check: "smart", status: "unavailable", reason: "tool_missing", detail: "smartctl" }, { check: "x", status: "bogus" }, "junk", { status: "failed" }] })).status,
|
|
202,
|
|
);
|
|
let d = (await detail()).diagnostics;
|
|
assert.deepEqual(d.map((n: { check: string }) => n.check), ["smart"]);
|
|
assert.match(d[0].fix, /smartmontools/);
|
|
|
|
assert.equal((await report({ cpu: { cores: 2 } })).status, 202);
|
|
assert.equal((await detail()).diagnostics.length, 1, "a report that doesn't say keeps what was said");
|
|
assert.equal((await report({ diagnostics: "garbage" })).status, 202);
|
|
assert.equal((await detail()).diagnostics.length, 1, "and so does one that isn't a list");
|
|
|
|
assert.equal((await report({ diagnostics: [] })).status, 202);
|
|
assert.deepEqual((await detail()).diagnostics, [], "an empty list says nothing was missed");
|
|
const row = (await (await fetch(`${base}/servers`)).json()).servers.find((x: { id: number }) => x.id === id);
|
|
assert.equal(row.agentDiagnostics, undefined, "never as raw stored text");
|
|
});
|
|
|
|
test("a service the server is told to ignore stops being an alert and a figure, still shows on the page marked, and can be un-ignored", async () => {
|
|
assert.equal((await report({ health: { failed_units: [{ unit: "nginx.service", description: "web" }, { unit: "gupdate.service", description: "Google Update" }] } })).status, 202);
|
|
invalidateAlertsCache();
|
|
const failing = async () => (await getAlerts(true)).alerts.find((a) => a.id.startsWith("services:failed-units") && a.id.includes(`:server:${id}`));
|
|
assert.match((await failing())!.message, /2 services have failed/);
|
|
|
|
assert.equal((await patch({ thresholds: { ignoreServices: ["gupdate*"] } })).status, 200);
|
|
assert.match((await failing())!.message, /1 service has failed — nginx\.service/, "the alert cache was dropped when the list changed");
|
|
assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\\{server="elsa"[^}]*\\} 1`));
|
|
|
|
const units = (await detail()).server.hostHealth.failedUnits;
|
|
assert.deepEqual(units.map((u: { unit: string; ignored?: boolean }) => [u.unit, u.ignored ?? false]), [["nginx.service", false], ["gupdate.service", true]]);
|
|
|
|
assert.equal((await patch({ thresholds: { ignoreServices: ["gupdate*", "nginx"] } })).status, 200);
|
|
assert.equal(await failing(), undefined, "with every failed service ignored there is nothing to alert about");
|
|
|
|
assert.equal((await patch({ thresholds: null })).status, 200);
|
|
assert.match((await failing())!.message, /2 services have failed/);
|
|
});
|
|
|
|
test("a list that isn't one is refused with a message", async () => {
|
|
const res = await patch({ thresholds: { ignoreServices: "gupdate" } });
|
|
assert.equal(res.status, 400);
|
|
assert.match((await res.json()).message, /should be a list/);
|
|
});
|
|
});
|