Host health from the Linux agent, agent self-update, and disk forecast in /metrics
- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates; SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox - Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series - Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified download, atomic replace; GET /api/agent/report/update - homelab_server_disk_days_to_full in /metrics - Tests (server, real PowerShell against a fake server) and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
540ab704b7
commit
5dc6cea69c
38 files changed
+3884
-73
No files matched your search
@@ -0,0 +1,316 @@
|
||||
/**
|
||||
* What an agent can say about a machine's health (failed services, disk SMART, ZFS pools), the alerts and figures that come from it, and
|
||||
* the switch that lets agents that are behind replace themselves.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-hosthealth-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, maintenanceWindows } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { hostHealthConditions, nextHostHealth, parseHostHealth } = await import("../src/services/hostHealth.js");
|
||||
const { evaluateHealth } = await import("../src/services/healthMonitor.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { latestAgentSha256, latestAgentVersions } = await import("../src/services/agentVersions.js");
|
||||
const { serversRouter } = await import("../src/routes/servers.js");
|
||||
const { agentReportRouter } = await import("../src/routes/agentReport.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const NOW = "2026-10-17T12:00:00.000Z";
|
||||
const health = (extra: object = {}) => ({ reportedAt: NOW, ...extra });
|
||||
|
||||
describe("what a health report amounts to", () => {
|
||||
test("failed services, a failing disk, a worn one and a pool that isn't online are each a problem; the healthy are not", () => {
|
||||
const conditions = hostHealthConditions(7, "elsa", {
|
||||
reportedAt: NOW,
|
||||
failedUnits: [{ unit: "nginx.service", description: "" }, { unit: "backup.service", description: "" }],
|
||||
smart: [
|
||||
{ device: "/dev/sda", model: "WD Red", healthy: false, temperatureC: 40, wearPercent: null },
|
||||
{ device: "/dev/nvme0", model: "Samsung", healthy: true, temperatureC: 38, wearPercent: 93 },
|
||||
{ device: "/dev/sdb", model: "Fine", healthy: true, temperatureC: 31, wearPercent: 12 },
|
||||
{ device: "/dev/sdc", model: "No verdict", healthy: null, temperatureC: null, wearPercent: null },
|
||||
],
|
||||
zfs: [{ name: "tank", health: "DEGRADED", capacityPercent: 40 }, { name: "rpool", health: "ONLINE", capacityPercent: 10 }],
|
||||
});
|
||||
const byKey = Object.fromEntries(conditions.map((c) => [c.key, c]));
|
||||
assert.deepEqual(Object.keys(byKey).sort(), ["failed-units:server:7", "smart:server:7:/dev/nvme0", "smart:server:7:/dev/sda", "zfs:server:7:tank"]);
|
||||
assert.equal(byKey["failed-units:server:7"].severity, "warning");
|
||||
assert.match(byKey["failed-units:server:7"].message, /2 services have failed — nginx\.service, backup\.service/);
|
||||
assert.equal(byKey["smart:server:7:/dev/sda"].severity, "critical");
|
||||
assert.match(byKey["smart:server:7:/dev/sda"].message, /failing/);
|
||||
assert.equal(byKey["smart:server:7:/dev/nvme0"].severity, "warning");
|
||||
assert.match(byKey["smart:server:7:/dev/nvme0"].message, /93% of its rated life/);
|
||||
assert.equal(byKey["zfs:server:7:tank"].severity, "critical");
|
||||
assert.match(byKey["zfs:server:7:tank"].message, /DEGRADED/);
|
||||
});
|
||||
|
||||
test("nothing said, or everything fine, is no problem; a long list of services is shortened", () => {
|
||||
assert.deepEqual(hostHealthConditions(1, "x", null), []);
|
||||
assert.deepEqual(hostHealthConditions(1, "x", health({ failedUnits: [], smart: [], zfs: [] })), []);
|
||||
const many = Array.from({ length: 9 }, (_, i) => ({ unit: `u${i}.service`, description: "" }));
|
||||
assert.match(hostHealthConditions(1, "x", health({ failedUnits: many }))[0].message, /and 5 more/);
|
||||
assert.match(hostHealthConditions(1, "x", health({ failedUnits: [many[0]] }))[0].message, /1 service has failed/);
|
||||
});
|
||||
|
||||
test("a report replaces the parts it carries and keeps the parts it doesn't", () => {
|
||||
const first = nextHostHealth(null, { failed_units: [{ unit: "a.service" }], zfs: [{ name: "tank", health: "ONLINE", capacity_percent: 20 }] }, "t1");
|
||||
assert.equal(parseHostHealth(first)!.failedUnits![0].unit, "a.service");
|
||||
// the next report knows about disks only
|
||||
const second = nextHostHealth(first, { smart: [{ device: "/dev/sda", healthy: true }] }, "t2");
|
||||
const merged = parseHostHealth(second)!;
|
||||
assert.equal(merged.failedUnits![0].unit, "a.service", "services kept");
|
||||
assert.equal(merged.zfs![0].name, "tank", "pools kept");
|
||||
assert.equal(merged.smart![0].device, "/dev/sda");
|
||||
assert.equal(merged.reportedAt, "t2");
|
||||
// an empty list is an answer ("none failed"), and clears the old one
|
||||
assert.deepEqual(parseHostHealth(nextHostHealth(second, { failed_units: [] }, "t3"))!.failedUnits, []);
|
||||
// nothing in the report: what was stored is untouched
|
||||
assert.equal(nextHostHealth(second, {}, "t4"), second);
|
||||
assert.equal(parseHostHealth("not json"), null);
|
||||
});
|
||||
|
||||
test("an offline server isn't judged on its health, and an online one is", () => {
|
||||
const snap = (lastSeenAt: string) => ({ servers: [{ id: 3, name: "elsa", lastSeenAt, disks: [], health: health({ failedUnits: [{ unit: "a.service", description: "" }] }) }], proxmox: [], synology: [] });
|
||||
const now = Date.parse(NOW);
|
||||
const limits = { serverOfflineMinutes: 60, diskUsagePercent: 90 };
|
||||
const online = evaluateHealth(snap(new Date(now - 60_000).toISOString()), limits, now).map((c) => c.key);
|
||||
assert.deepEqual(online, ["failed-units:server:3"]);
|
||||
const offline = evaluateHealth(snap(new Date(now - 5 * 3600_000).toISOString()), limits, now).map((c) => c.key);
|
||||
assert.deepEqual(offline, ["offline:server:3"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the report, the alerts, the figures and the page", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let id = 0;
|
||||
let who = "admin";
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olle", role: "operator" },
|
||||
]);
|
||||
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t" }).returning();
|
||||
id = s.id;
|
||||
const a = express();
|
||||
a.use("/api/agent/report", agentReportRouter);
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/servers", serversRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const report = (system: unknown, extra: Record<string, unknown> = {}) =>
|
||||
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", tasks: [], system, ...extra }) });
|
||||
const detail = async () => (await (await fetch(`${base}/servers/${id}/detail`)).json()).server;
|
||||
|
||||
test("what an agent says is stored and shown; a garbled part costs the report nothing", async () => {
|
||||
const res = await report({
|
||||
health: {
|
||||
failed_units: [{ unit: "nginx.service", description: "A high performance web server" }],
|
||||
smart: [{ device: "/dev/sda", model: "WD Red", healthy: false, temperature_c: 41, wear_percent: null }],
|
||||
zfs: "this isn't a list",
|
||||
},
|
||||
});
|
||||
assert.equal(res.status, 202);
|
||||
let s = await detail();
|
||||
assert.equal(s.hostHealth.failedUnits[0].unit, "nginx.service");
|
||||
assert.equal(s.hostHealth.smart[0].healthy, false);
|
||||
assert.equal(s.hostHealth.zfs, undefined, "the part that made no sense was dropped");
|
||||
// the list endpoint shows it too, and never as raw stored text
|
||||
const row = (await (await fetch(`${base}/servers`)).json()).servers.find((x: { id: number }) => x.id === id);
|
||||
assert.equal(typeof row.hostHealth, "object");
|
||||
|
||||
// a report with no health at all leaves it as it was; garbage in place of the whole thing too
|
||||
assert.equal((await report({ cpu: { cores: 2 } })).status, 202);
|
||||
assert.equal((await report({ health: 12 })).status, 202);
|
||||
s = await detail();
|
||||
assert.equal(s.hostHealth.failedUnits[0].unit, "nginx.service");
|
||||
});
|
||||
|
||||
test("they appear on the Alerts page as their own kind, under the server, and a maintenance window silences them", async () => {
|
||||
invalidateAlertsCache();
|
||||
let alerts = (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`));
|
||||
const services = alerts.find((a) => a.id.startsWith("services:failed-units"));
|
||||
assert.ok(services, JSON.stringify(alerts.map((a) => a.id)));
|
||||
assert.equal(services.category, "services");
|
||||
assert.equal(services.link, `/servers/${id}`);
|
||||
assert.equal(services.silenced, false);
|
||||
const disk = alerts.find((a) => a.id.startsWith("disk:smart"));
|
||||
assert.ok(disk && disk.severity === "critical", "a failing disk is critical, in the disk category");
|
||||
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: id, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
|
||||
invalidateAlertsCache();
|
||||
alerts = (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`));
|
||||
assert.ok(alerts.length >= 2 && alerts.every((a) => a.silenced), "all held back under maintenance");
|
||||
await db.delete(maintenanceWindows).where(eq(maintenanceWindows.targetId, id));
|
||||
|
||||
// the services come right once the agent says none have failed
|
||||
await report({ health: { failed_units: [] } });
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await getAlerts(true)).alerts.some((a) => a.id.startsWith("services:")), false);
|
||||
});
|
||||
|
||||
test("/metrics carries them", async () => {
|
||||
await report({ health: { failed_units: [{ unit: "a.service" }, { unit: "b.service" }], zfs: [{ name: "tank", health: "DEGRADED", capacity_percent: 12 }], smart: [{ device: "/dev/sda", healthy: true, temperature_c: 33, wear_percent: 5 }] } });
|
||||
const text = await buildMetrics();
|
||||
assert.match(text, /homelab_server_failed_units\{server="elsa"\} 2/);
|
||||
assert.match(text, /homelab_server_zfs_pool_online\{server="elsa",pool="tank"\} 0/);
|
||||
assert.match(text, /homelab_server_disk_healthy\{server="elsa",device="\/dev\/sda"\} 1/);
|
||||
assert.match(text, /homelab_server_disk_wear_percent\{server="elsa",device="\/dev\/sda"\} 5/);
|
||||
assert.match(text, /homelab_server_disk_temperature_celsius\{server="elsa",device="\/dev\/sda"\} 33/);
|
||||
});
|
||||
|
||||
test("a disk on a steady climb has its days to full in /metrics, and a flat one does not", async () => {
|
||||
const { serverMetrics } = await import("../src/db/schema.js");
|
||||
const day = 86_400_000;
|
||||
const rows = [];
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const at = new Date(Date.now() - (10 - i) * day).toISOString();
|
||||
rows.push({ serverId: id, metric: "disk:/data", value: 50 + i * 2, recordedAt: at });
|
||||
rows.push({ serverId: id, metric: "disk:/", value: 30, recordedAt: at });
|
||||
}
|
||||
await db.insert(serverMetrics).values(rows);
|
||||
const text = await buildMetrics();
|
||||
const m = /homelab_server_disk_days_to_full\{server="elsa",mount="\/data"\} (\d+)/.exec(text);
|
||||
assert.ok(m, text.split("\n").filter((l) => l.includes("days_to_full")).join("\n"));
|
||||
assert.ok(Number(m[1]) >= 15 && Number(m[1]) <= 17, `about sixteen days (2 points a day, 32 left from 68%), got ${m[1]}`);
|
||||
assert.doesNotMatch(text, /days_to_full\{server="elsa",mount="\/"\}/);
|
||||
});
|
||||
|
||||
test("an agent that is now a newer version leaves a line in the server's history", async () => {
|
||||
await report({}, { agent_version: 3 });
|
||||
await report({}, { agent_version: 4 });
|
||||
const activity = await (await fetch(`${base}/servers/${id}/activity`)).json();
|
||||
const line = activity.events.find((e: { title: string }) => /Agent updated to version 4/.test(e.title));
|
||||
assert.ok(line, JSON.stringify(activity.events.map((e: { title: string }) => e.title)));
|
||||
assert.equal(line.detail, "from version 3");
|
||||
assert.equal(activity.events.some((e: { title: string }) => /version 3$/.test(e.title)), false, "the first time a version is seen is not an update");
|
||||
});
|
||||
});
|
||||
|
||||
describe("agents updating themselves", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
let id = 0;
|
||||
|
||||
before(async () => {
|
||||
const [s] = await db.insert(servers).values({ name: "anna", apiTokenHash: hashToken("anna-tok"), apiTokenPrefix: "a", osType: "linux" }).returning();
|
||||
id = s.id;
|
||||
const [w] = await db.insert(servers).values({ name: "win", apiTokenHash: hashToken("win-tok"), apiTokenPrefix: "w", osType: "windows", agentVersion: 1 }).returning();
|
||||
void w;
|
||||
const a = express();
|
||||
a.use("/api/agent/report", agentReportRouter);
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/servers", serversRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
const ask = (token: string | null, query = "") => fetch(`${base}/agent/report/update${query}`, { headers: token ? { authorization: `Bearer ${token}` } : {} });
|
||||
const put = (autoUpdate: unknown) => fetch(`${base}/servers/agent-update`, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ autoUpdate }) });
|
||||
|
||||
test("only an agent that proves who it is may ask", async () => {
|
||||
assert.equal((await ask(null)).status, 401);
|
||||
assert.equal((await ask("nobody")).status, 401);
|
||||
});
|
||||
|
||||
test("while the switch is off nobody is told to update, however far behind", async () => {
|
||||
const list = await (await fetch(`${base}/servers`)).json();
|
||||
assert.equal(list.agentAutoUpdate, false, "off until an administrator turns it on");
|
||||
const reply = await (await ask("anna-tok", "?version=1")).json();
|
||||
assert.deepEqual(reply, { update: false, version: null, sha256: null });
|
||||
});
|
||||
|
||||
test("only an administrator can switch it, it is written in the audit log, and it shows on the Servers list", async () => {
|
||||
who = "operator";
|
||||
assert.equal((await put(true)).status, 403);
|
||||
who = "admin";
|
||||
assert.equal((await put("yes")).status, 400);
|
||||
assert.equal((await put(true)).status, 200);
|
||||
assert.equal((await (await fetch(`${base}/servers`)).json()).agentAutoUpdate, true);
|
||||
const { auditLog } = await import("../src/db/schema.js");
|
||||
const entry = (await db.select().from(auditLog)).find((e) => e.action === "agent_auto_update");
|
||||
assert.ok(entry);
|
||||
assert.deepEqual(JSON.parse(entry.detail!), { from: false, to: true });
|
||||
});
|
||||
|
||||
test("with it on, an agent that is behind is told to, with the checksum of what it will fetch; one that is current is not", async () => {
|
||||
const latest = latestAgentVersions().linux!;
|
||||
const behind = await (await ask("anna-tok", `?version=${latest - 1}`)).json();
|
||||
assert.equal(behind.update, true);
|
||||
assert.equal(behind.version, latest);
|
||||
assert.equal(behind.sha256, latestAgentSha256("linux"));
|
||||
assert.match(behind.sha256, /^[0-9a-f]{64}$/);
|
||||
|
||||
const current = await (await ask("anna-tok", `?version=${latest}`)).json();
|
||||
assert.deepEqual(current, { update: false, version: null, sha256: null });
|
||||
const ahead = await (await ask("anna-tok", `?version=${latest + 5}`)).json();
|
||||
assert.equal(ahead.update, false);
|
||||
|
||||
// no version given: what it last reported counts, and an agent that has said nothing is version 1
|
||||
assert.equal((await (await ask("anna-tok")).json()).update, true);
|
||||
await db.update(servers).set({ agentVersion: latest }).where(eq(servers.id, id));
|
||||
assert.equal((await (await ask("anna-tok")).json()).update, false);
|
||||
// nonsense is ignored in favour of that
|
||||
assert.equal((await (await ask("anna-tok", "?version=banana")).json()).update, false);
|
||||
});
|
||||
|
||||
test("each system is given its own script's checksum", async () => {
|
||||
const reply = await (await ask("win-tok", "?version=1")).json();
|
||||
assert.equal(reply.update, true);
|
||||
assert.equal(reply.sha256, latestAgentSha256("windows"));
|
||||
assert.notEqual(reply.sha256, latestAgentSha256("linux"));
|
||||
});
|
||||
|
||||
test("the checksum is of the file that is served, byte for byte", async () => {
|
||||
const { createHash } = await import("node:crypto");
|
||||
const { readFileSync } = await import("node:fs");
|
||||
const { AGENT_DIR } = await import("../src/services/agentVersions.js");
|
||||
const bytes = readFileSync(join(AGENT_DIR, "linux", "report-tasks.sh"));
|
||||
assert.equal(latestAgentSha256("linux"), createHash("sha256").update(bytes).digest("hex"));
|
||||
assert.equal(latestAgentSha256("linux", join(dir, "empty")), null, "an unreadable script has no checksum, so nobody is told to fetch it");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user