UniFi: the controller's client list is matched to servers by IP address, so a server is connected through a switch or access point (and port). That is a dependency made by itself, shown on the server page, the Network devices page and the Connections map. IP Addresses can sync the devices' addresses. A server's page is split into Overview, Health, Network, History, Tasks and Settings, by ?view=; alerts link to the section they are about. My work lists what has your name on it: alerts you have, maintenance you started (extend, end with undo) and update runs you started, with a badge. Settings has a search box (and the command palette lists every setting). Removing a dependency, admin link, runbook, hand-made IP address, port opening or restore test is undone from the message instead of asked about first. Two backup tests used a fixed date and failed as days passed; they follow the real clock now. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
317 lines
17 KiB
TypeScript
317 lines
17 KiB
TypeScript
/**
|
|
* What an agent can say about a machine's health (failed services, disk SMART, ZFS pools), the alerts and figures that come from it, and
|
|
* the switch that lets agents that are behind replace themselves.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { AddressInfo } from "node:net";
|
|
import type { Server } from "node:http";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-hosthealth-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const express = (await import("express")).default;
|
|
const { db } = await import("../src/db/client.js");
|
|
const { users, servers, maintenanceWindows } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const { hostHealthConditions, nextHostHealth, parseHostHealth } = await import("../src/services/hostHealth.js");
|
|
const { evaluateHealth } = await import("../src/services/healthMonitor.js");
|
|
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
|
const { buildMetrics } = await import("../src/services/metrics.js");
|
|
const { latestAgentSha256, latestAgentVersions } = await import("../src/services/agentVersions.js");
|
|
const { serversRouter } = await import("../src/routes/servers.js");
|
|
const { agentReportRouter } = await import("../src/routes/agentReport.js");
|
|
const { hashToken } = await import("../src/services/tokens.js");
|
|
const { eq } = await import("drizzle-orm");
|
|
|
|
await runMigrations();
|
|
|
|
const NOW = "2026-10-17T12:00:00.000Z";
|
|
const health = (extra: object = {}) => ({ reportedAt: NOW, ...extra });
|
|
|
|
describe("what a health report amounts to", () => {
|
|
test("failed services, a failing disk, a worn one and a pool that isn't online are each a problem; the healthy are not", () => {
|
|
const conditions = hostHealthConditions(7, "elsa", {
|
|
reportedAt: NOW,
|
|
failedUnits: [{ unit: "nginx.service", description: "" }, { unit: "backup.service", description: "" }],
|
|
smart: [
|
|
{ device: "/dev/sda", model: "WD Red", healthy: false, temperatureC: 40, wearPercent: null },
|
|
{ device: "/dev/nvme0", model: "Samsung", healthy: true, temperatureC: 38, wearPercent: 93 },
|
|
{ device: "/dev/sdb", model: "Fine", healthy: true, temperatureC: 31, wearPercent: 12 },
|
|
{ device: "/dev/sdc", model: "No verdict", healthy: null, temperatureC: null, wearPercent: null },
|
|
],
|
|
zfs: [{ name: "tank", health: "DEGRADED", capacityPercent: 40 }, { name: "rpool", health: "ONLINE", capacityPercent: 10 }],
|
|
});
|
|
const byKey = Object.fromEntries(conditions.map((c) => [c.key, c]));
|
|
assert.deepEqual(Object.keys(byKey).sort(), ["failed-units:server:7", "smart:server:7:/dev/nvme0", "smart:server:7:/dev/sda", "zfs:server:7:tank"]);
|
|
assert.equal(byKey["failed-units:server:7"].severity, "warning");
|
|
assert.match(byKey["failed-units:server:7"].message, /2 services have failed — nginx\.service, backup\.service/);
|
|
assert.equal(byKey["smart:server:7:/dev/sda"].severity, "critical");
|
|
assert.match(byKey["smart:server:7:/dev/sda"].message, /failing/);
|
|
assert.equal(byKey["smart:server:7:/dev/nvme0"].severity, "warning");
|
|
assert.match(byKey["smart:server:7:/dev/nvme0"].message, /93% of its rated life/);
|
|
assert.equal(byKey["zfs:server:7:tank"].severity, "critical");
|
|
assert.match(byKey["zfs:server:7:tank"].message, /DEGRADED/);
|
|
});
|
|
|
|
test("nothing said, or everything fine, is no problem; a long list of services is shortened", () => {
|
|
assert.deepEqual(hostHealthConditions(1, "x", null), []);
|
|
assert.deepEqual(hostHealthConditions(1, "x", health({ failedUnits: [], smart: [], zfs: [] })), []);
|
|
const many = Array.from({ length: 9 }, (_, i) => ({ unit: `u${i}.service`, description: "" }));
|
|
assert.match(hostHealthConditions(1, "x", health({ failedUnits: many }))[0].message, /and 5 more/);
|
|
assert.match(hostHealthConditions(1, "x", health({ failedUnits: [many[0]] }))[0].message, /1 service has failed/);
|
|
});
|
|
|
|
test("a report replaces the parts it carries and keeps the parts it doesn't", () => {
|
|
const first = nextHostHealth(null, { failed_units: [{ unit: "a.service" }], zfs: [{ name: "tank", health: "ONLINE", capacity_percent: 20 }] }, "t1");
|
|
assert.equal(parseHostHealth(first)!.failedUnits![0].unit, "a.service");
|
|
// the next report knows about disks only
|
|
const second = nextHostHealth(first, { smart: [{ device: "/dev/sda", healthy: true }] }, "t2");
|
|
const merged = parseHostHealth(second)!;
|
|
assert.equal(merged.failedUnits![0].unit, "a.service", "services kept");
|
|
assert.equal(merged.zfs![0].name, "tank", "pools kept");
|
|
assert.equal(merged.smart![0].device, "/dev/sda");
|
|
assert.equal(merged.reportedAt, "t2");
|
|
// an empty list is an answer ("none failed"), and clears the old one
|
|
assert.deepEqual(parseHostHealth(nextHostHealth(second, { failed_units: [] }, "t3"))!.failedUnits, []);
|
|
// nothing in the report: what was stored is untouched
|
|
assert.equal(nextHostHealth(second, {}, "t4"), second);
|
|
assert.equal(parseHostHealth("not json"), null);
|
|
});
|
|
|
|
test("an offline server isn't judged on its health, and an online one is", () => {
|
|
const snap = (lastSeenAt: string) => ({ servers: [{ id: 3, name: "elsa", lastSeenAt, disks: [], health: health({ failedUnits: [{ unit: "a.service", description: "" }] }) }], proxmox: [], synology: [] });
|
|
const now = Date.parse(NOW);
|
|
const limits = { serverOfflineMinutes: 60, diskUsagePercent: 90 };
|
|
const online = evaluateHealth(snap(new Date(now - 60_000).toISOString()), limits, now).map((c) => c.key);
|
|
assert.deepEqual(online, ["failed-units:server:3"]);
|
|
const offline = evaluateHealth(snap(new Date(now - 5 * 3600_000).toISOString()), limits, now).map((c) => c.key);
|
|
assert.deepEqual(offline, ["offline:server:3"]);
|
|
});
|
|
});
|
|
|
|
describe("through the report, the alerts, the figures and the page", () => {
|
|
let app: Server;
|
|
let base = "";
|
|
let id = 0;
|
|
let who = "admin";
|
|
|
|
before(async () => {
|
|
await db.insert(users).values([
|
|
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
|
{ oidcSub: "operator", name: "Olle", role: "operator" },
|
|
]);
|
|
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t" }).returning();
|
|
id = s.id;
|
|
const a = express();
|
|
a.use("/api/agent/report", agentReportRouter);
|
|
a.use(express.json());
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
|
next();
|
|
});
|
|
a.use("/api/servers", serversRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
|
});
|
|
|
|
after(() => {
|
|
app.close();
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows keeps the database file locked until the process exits.
|
|
}
|
|
});
|
|
|
|
const report = (system: unknown, extra: Record<string, unknown> = {}) =>
|
|
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", tasks: [], system, ...extra }) });
|
|
const detail = async () => (await (await fetch(`${base}/servers/${id}/detail`)).json()).server;
|
|
|
|
test("what an agent says is stored and shown; a garbled part costs the report nothing", async () => {
|
|
const res = await report({
|
|
health: {
|
|
failed_units: [{ unit: "nginx.service", description: "A high performance web server" }],
|
|
smart: [{ device: "/dev/sda", model: "WD Red", healthy: false, temperature_c: 41, wear_percent: null }],
|
|
zfs: "this isn't a list",
|
|
},
|
|
});
|
|
assert.equal(res.status, 202);
|
|
let s = await detail();
|
|
assert.equal(s.hostHealth.failedUnits[0].unit, "nginx.service");
|
|
assert.equal(s.hostHealth.smart[0].healthy, false);
|
|
assert.equal(s.hostHealth.zfs, undefined, "the part that made no sense was dropped");
|
|
// the list endpoint shows it too, and never as raw stored text
|
|
const row = (await (await fetch(`${base}/servers`)).json()).servers.find((x: { id: number }) => x.id === id);
|
|
assert.equal(typeof row.hostHealth, "object");
|
|
|
|
// a report with no health at all leaves it as it was; garbage in place of the whole thing too
|
|
assert.equal((await report({ cpu: { cores: 2 } })).status, 202);
|
|
assert.equal((await report({ health: 12 })).status, 202);
|
|
s = await detail();
|
|
assert.equal(s.hostHealth.failedUnits[0].unit, "nginx.service");
|
|
});
|
|
|
|
test("they appear on the Alerts page as their own kind, under the server, and a maintenance window silences them", async () => {
|
|
invalidateAlertsCache();
|
|
let alerts = (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`));
|
|
const services = alerts.find((a) => a.id.startsWith("services:failed-units"));
|
|
assert.ok(services, JSON.stringify(alerts.map((a) => a.id)));
|
|
assert.equal(services.category, "services");
|
|
assert.equal(services.link, `/servers/${id}?view=health`, "it opens the Health section of the server page");
|
|
assert.equal(services.silenced, false);
|
|
const disk = alerts.find((a) => a.id.startsWith("disk:smart"));
|
|
assert.ok(disk && disk.severity === "critical", "a failing disk is critical, in the disk category");
|
|
|
|
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: id, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
|
|
invalidateAlertsCache();
|
|
alerts = (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`));
|
|
assert.ok(alerts.length >= 2 && alerts.every((a) => a.silenced), "all held back under maintenance");
|
|
await db.delete(maintenanceWindows).where(eq(maintenanceWindows.targetId, id));
|
|
|
|
// the services come right once the agent says none have failed
|
|
await report({ health: { failed_units: [] } });
|
|
invalidateAlertsCache();
|
|
assert.equal((await getAlerts(true)).alerts.some((a) => a.id.startsWith("services:")), false);
|
|
});
|
|
|
|
test("/metrics carries them", async () => {
|
|
await report({ health: { failed_units: [{ unit: "a.service" }, { unit: "b.service" }], zfs: [{ name: "tank", health: "DEGRADED", capacity_percent: 12 }], smart: [{ device: "/dev/sda", healthy: true, temperature_c: 33, wear_percent: 5 }] } });
|
|
const text = await buildMetrics();
|
|
assert.match(text, /homelab_server_failed_units\{server="elsa"\} 2/);
|
|
assert.match(text, /homelab_server_zfs_pool_online\{server="elsa",pool="tank"\} 0/);
|
|
assert.match(text, /homelab_server_disk_healthy\{server="elsa",device="\/dev\/sda"\} 1/);
|
|
assert.match(text, /homelab_server_disk_wear_percent\{server="elsa",device="\/dev\/sda"\} 5/);
|
|
assert.match(text, /homelab_server_disk_temperature_celsius\{server="elsa",device="\/dev\/sda"\} 33/);
|
|
});
|
|
|
|
test("a disk on a steady climb has its days to full in /metrics, and a flat one does not", async () => {
|
|
const { serverMetrics } = await import("../src/db/schema.js");
|
|
const day = 86_400_000;
|
|
const rows = [];
|
|
for (let i = 0; i < 10; i++) {
|
|
const at = new Date(Date.now() - (10 - i) * day).toISOString();
|
|
rows.push({ serverId: id, metric: "disk:/data", value: 50 + i * 2, recordedAt: at });
|
|
rows.push({ serverId: id, metric: "disk:/", value: 30, recordedAt: at });
|
|
}
|
|
await db.insert(serverMetrics).values(rows);
|
|
const text = await buildMetrics();
|
|
const m = /homelab_server_disk_days_to_full\{server="elsa",mount="\/data"\} (\d+)/.exec(text);
|
|
assert.ok(m, text.split("\n").filter((l) => l.includes("days_to_full")).join("\n"));
|
|
assert.ok(Number(m[1]) >= 15 && Number(m[1]) <= 17, `about sixteen days (2 points a day, 32 left from 68%), got ${m[1]}`);
|
|
assert.doesNotMatch(text, /days_to_full\{server="elsa",mount="\/"\}/);
|
|
});
|
|
|
|
test("an agent that is now a newer version leaves a line in the server's history", async () => {
|
|
await report({}, { agent_version: 3 });
|
|
await report({}, { agent_version: 4 });
|
|
const activity = await (await fetch(`${base}/servers/${id}/activity`)).json();
|
|
const line = activity.events.find((e: { title: string }) => /Agent updated to version 4/.test(e.title));
|
|
assert.ok(line, JSON.stringify(activity.events.map((e: { title: string }) => e.title)));
|
|
assert.equal(line.detail, "from version 3");
|
|
assert.equal(activity.events.some((e: { title: string }) => /version 3$/.test(e.title)), false, "the first time a version is seen is not an update");
|
|
});
|
|
});
|
|
|
|
describe("agents updating themselves", () => {
|
|
let app: Server;
|
|
let base = "";
|
|
let who = "admin";
|
|
let id = 0;
|
|
|
|
before(async () => {
|
|
const [s] = await db.insert(servers).values({ name: "anna", apiTokenHash: hashToken("anna-tok"), apiTokenPrefix: "a", osType: "linux" }).returning();
|
|
id = s.id;
|
|
const [w] = await db.insert(servers).values({ name: "win", apiTokenHash: hashToken("win-tok"), apiTokenPrefix: "w", osType: "windows", agentVersion: 1 }).returning();
|
|
void w;
|
|
const a = express();
|
|
a.use("/api/agent/report", agentReportRouter);
|
|
a.use(express.json());
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
|
next();
|
|
});
|
|
a.use("/api/servers", serversRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
|
});
|
|
after(() => app.close());
|
|
|
|
const ask = (token: string | null, query = "") => fetch(`${base}/agent/report/update${query}`, { headers: token ? { authorization: `Bearer ${token}` } : {} });
|
|
const put = (autoUpdate: unknown) => fetch(`${base}/servers/agent-update`, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ autoUpdate }) });
|
|
|
|
test("only an agent that proves who it is may ask", async () => {
|
|
assert.equal((await ask(null)).status, 401);
|
|
assert.equal((await ask("nobody")).status, 401);
|
|
});
|
|
|
|
test("while the switch is off nobody is told to update, however far behind", async () => {
|
|
const list = await (await fetch(`${base}/servers`)).json();
|
|
assert.equal(list.agentAutoUpdate, false, "off until an administrator turns it on");
|
|
const reply = await (await ask("anna-tok", "?version=1")).json();
|
|
assert.deepEqual(reply, { update: false, version: null, sha256: null });
|
|
});
|
|
|
|
test("only an administrator can switch it, it is written in the audit log, and it shows on the Servers list", async () => {
|
|
who = "operator";
|
|
assert.equal((await put(true)).status, 403);
|
|
who = "admin";
|
|
assert.equal((await put("yes")).status, 400);
|
|
assert.equal((await put(true)).status, 200);
|
|
assert.equal((await (await fetch(`${base}/servers`)).json()).agentAutoUpdate, true);
|
|
const { auditLog } = await import("../src/db/schema.js");
|
|
const entry = (await db.select().from(auditLog)).find((e) => e.action === "agent_auto_update");
|
|
assert.ok(entry);
|
|
assert.deepEqual(JSON.parse(entry.detail!), { from: false, to: true });
|
|
});
|
|
|
|
test("with it on, an agent that is behind is told to, with the checksum of what it will fetch; one that is current is not", async () => {
|
|
const latest = latestAgentVersions().linux!;
|
|
const behind = await (await ask("anna-tok", `?version=${latest - 1}`)).json();
|
|
assert.equal(behind.update, true);
|
|
assert.equal(behind.version, latest);
|
|
assert.equal(behind.sha256, latestAgentSha256("linux"));
|
|
assert.match(behind.sha256, /^[0-9a-f]{64}$/);
|
|
|
|
const current = await (await ask("anna-tok", `?version=${latest}`)).json();
|
|
assert.deepEqual(current, { update: false, version: null, sha256: null });
|
|
const ahead = await (await ask("anna-tok", `?version=${latest + 5}`)).json();
|
|
assert.equal(ahead.update, false);
|
|
|
|
// no version given: what it last reported counts, and an agent that has said nothing is version 1
|
|
assert.equal((await (await ask("anna-tok")).json()).update, true);
|
|
await db.update(servers).set({ agentVersion: latest }).where(eq(servers.id, id));
|
|
assert.equal((await (await ask("anna-tok")).json()).update, false);
|
|
// nonsense is ignored in favour of that
|
|
assert.equal((await (await ask("anna-tok", "?version=banana")).json()).update, false);
|
|
});
|
|
|
|
test("each system is given its own script's checksum", async () => {
|
|
const reply = await (await ask("win-tok", "?version=1")).json();
|
|
assert.equal(reply.update, true);
|
|
assert.equal(reply.sha256, latestAgentSha256("windows"));
|
|
assert.notEqual(reply.sha256, latestAgentSha256("linux"));
|
|
});
|
|
|
|
test("the checksum is of the file that is served, byte for byte", async () => {
|
|
const { createHash } = await import("node:crypto");
|
|
const { readFileSync } = await import("node:fs");
|
|
const { AGENT_DIR } = await import("../src/services/agentVersions.js");
|
|
const bytes = readFileSync(join(AGENT_DIR, "linux", "report-tasks.sh"));
|
|
assert.equal(latestAgentSha256("linux"), createHash("sha256").update(bytes).digest("hex"));
|
|
assert.equal(latestAgentSha256("linux", join(dir, "empty")), null, "an unreadable script has no checksum, so nobody is told to fetch it");
|
|
});
|
|
});
|