Host health from the Linux agent, agent self-update, and disk forecast in /metrics

- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates;
  SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox
- Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new
  "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series
- Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified
  download, atomic replace; GET /api/agent/report/update
- homelab_server_disk_days_to_full in /metrics
- Tests (server, real PowerShell against a fake server) and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 00:54:24 +02:00
1 parent 540ab704b7
commit 5dc6cea69c
38 files changed
+3884 -73

No files matched your search

@@ -0,0 +1 @@
ALTER TABLE `servers` ADD `host_health` text;
File diff suppressed because it is too large. Load diff
+7
View File
@@ -204,6 +204,13 @@
"when": 1792267930026,
"tag": "0028_thankful_ultron",
"breakpoints": true
},
{
"idx": 29,
"version": "6",
"when": 1792273577264,
"tag": "0029_past_monster_badoon",
"breakpoints": true
}
]
}
+3
View File
@@ -297,6 +297,9 @@ export const servers = sqliteTable("servers", {
// When updates were first seen waiting (and none were before) — how long the oldest has been waiting. Null when none are.
updatesPendingSince: text("updates_pending_since"),
// What the agent says about the machine's own health, as JSON: {failedUnits, smart, zfs, reportedAt} — see services/hostHealth.ts. Null until it says.
hostHealth: text("host_health"),
// Which version of the agent script is running on it, as the agent says in each report (see agent/*/report-tasks.*). Null until it says:
// an agent that predates the number being reported is version 1.
agentVersion: integer("agent_version"),
+45
View File
@@ -4,6 +4,8 @@ import { db } from "../db/client.js";
import { hashToken } from "../services/tokens.js";
import { syncServerTasks } from "../services/taskSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { getSettings } from "../services/settingsStore.js";
import { latestAgentSha256, latestAgentVersions } from "../services/agentVersions.js";
export const agentReportRouter = Router();
@@ -31,7 +33,33 @@ const updatesSchema = z.object({
reboot_required: z.boolean().optional(),
});
// Failed services, disk health, ZFS pools. Each part is checked on its own and a bad one dropped, like the rest: it must never cost the report.
const healthSchema = z.object({
failed_units: z
.array(z.object({ unit: text(200), description: text(300).optional() }))
.transform(cap(100))
.optional()
.catch(undefined),
smart: z
.array(z.object({ device: text(100), model: text(200).optional(), healthy: z.boolean().nullable().optional(), temperature_c: z.number().nullable().optional(), wear_percent: z.number().nullable().optional() }))
.transform(cap(100))
.optional()
.catch(undefined),
zfs: z
.array(z.object({ name: text(200), health: text(40), capacity_percent: z.number().nullable().optional() }))
.transform(cap(100))
.optional()
.catch(undefined),
});
const systemSchema = z.object({
health: z
.unknown()
.optional()
.transform((h) => {
const parsed = healthSchema.safeParse(h);
return parsed.success && Object.values(parsed.data).some((v) => v !== undefined) ? parsed.data : undefined;
}),
updates: z
.unknown()
.optional()
@@ -105,6 +133,23 @@ async function authenticateAgent(req: Request, res: Response, next: NextFunction
next();
}
/**
* "Should I replace myself?" — asked by an agent that has been told to keep itself up to date, before it reports. Says yes only when an
* administrator has switched automatic updates on and this agent is behind the script this app serves, and gives the script's SHA-256 so
* the agent can check what it downloads. The agent says which version it is; failing that, the last one it reported counts.
*/
agentReportRouter.get("/update", asyncHandler(authenticateAgent), asyncHandler(async (req, res) => {
const server = res.locals.agentServer as { osType: string; agentVersion: number | null };
const os = server.osType === "windows" ? "windows" : "linux";
const claimed = Number(req.query.version);
const running = Number.isInteger(claimed) && claimed >= 1 && claimed <= 100_000 ? claimed : (server.agentVersion ?? 1);
const latest = latestAgentVersions()[os];
const sha256 = latestAgentSha256(os);
const { agents } = await getSettings();
const update = agents.autoUpdate && latest !== null && sha256 !== null && running < latest;
res.json({ update, version: update ? latest : null, sha256: update ? sha256 : null });
}));
agentReportRouter.post("/", asyncHandler(authenticateAgent), express.json({ limit: AGENT_BODY_LIMIT }), asyncHandler(async (req, res) => {
const server = res.locals.agentServer as { id: number };
+1
View File
@@ -20,6 +20,7 @@ alertTicketsRouter.use(requireAuth, requireRole("admin"));
const CATEGORIES = [
{ value: "offline", label: "Server down" },
{ value: "disk", label: "Disk & storage" },
{ value: "services", label: "Failing services" },
{ value: "backup", label: "Backups" },
{ value: "updates", label: "Updates" },
{ value: "expiry", label: "Expiry" },
+1 -1
View File
@@ -15,7 +15,7 @@ runbooksRouter.use(requireAuth);
/** More than anyone needs for one thing; each is another note to keep true. */
const MAX_PER_TARGET = 20;
export const ALERT_CATEGORIES = ["offline", "disk", "backup", "updates", "expiry", "automation", "integration", "monitoring", "tickets"] as const;
export const ALERT_CATEGORIES = ["offline", "disk", "services", "backup", "updates", "expiry", "automation", "integration", "monitoring", "tickets"] as const;
const bodySchema = z.object({
title: z.string().trim().min(1).max(120),
+19 -5
View File
@@ -8,6 +8,7 @@ import { generateApiToken } from "../services/tokens.js";
import { recordAudit } from "../services/audit.js";
import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js";
import { parsePendingUpdates } from "../services/pendingUpdates.js";
import { parseHostHealth } from "../services/hostHealth.js";
import { latestAgentVersions } from "../services/agentVersions.js";
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
import { getServerActivity } from "../services/serverActivity.js";
@@ -19,14 +20,14 @@ import { serverPortsRouter } from "./serverPorts.js";
import { InvalidTagError, normalizeTags, parseTags } from "../services/serverTags.js";
import { buildServerSummary } from "../services/serverSummary.js";
import { activeSubjects } from "../services/maintenance.js";
import { getSettings } from "../services/settingsStore.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
export const serversRouter = Router();
/** A server row as the API returns it: no token hash, and tags as an array rather than the stored JSON. */
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null }>(row: T) {
const { apiTokenHash: _hash, tags, pendingUpdates, ...rest } = row;
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates) };
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null; hostHealth: string | null }>(row: T) {
const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, ...rest } = row;
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth) };
}
serversRouter.use(requireAuth);
@@ -43,11 +44,23 @@ serversRouter.get("/", asyncHandler(async (_req, res) => {
const rows = await db.query.servers.findMany({ orderBy: (s, { asc }) => [asc(s.name)] });
res.json({
servers: rows.map(publicServer),
// Whether agents that are behind replace themselves, so the page can show the switch.
agentAutoUpdate: (await getSettings()).agents.autoUpdate,
// What a new install would be, so the page can say which agents are behind.
agentLatest: latestAgentVersions(),
});
}));
// Whether agents that are behind download the newer script from here and replace themselves. An administrator's to switch.
serversRouter.put("/agent-update", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = z.object({ autoUpdate: z.boolean() }).safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body" });
const before = (await getSettings()).agents.autoUpdate;
await updateSettings({ agents: { autoUpdate: parsed.data.autoUpdate } });
await recordAudit({ actor: req.currentUser!, category: "settings", action: "agent_auto_update", targetType: "settings", detail: { from: before, to: parsed.data.autoUpdate } });
res.json({ autoUpdate: parsed.data.autoUpdate });
}));
serversRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = createServerSchema.safeParse(req.body);
if (!parsed.success) {
@@ -353,6 +366,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
listeningPorts: _listeningPorts,
lastPortScan: _lastPortScan,
pendingUpdates: rawUpdates,
hostHealth: rawHealth,
...serverOut
} = server;
@@ -366,7 +380,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
children: [...links2.entries()].filter(([, l]) => l.parentId === id).map(([childId, l]) => ({ id: childId, name: nameOf.get(childId) ?? "", auto: l.auto })).sort((a, b) => a.name.localeCompare(b.name)),
};
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates) }, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: parseHostHealth(rawHealth) }, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
}));
const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) });
+16 -1
View File
@@ -6,8 +6,11 @@
*
* 1 — before the number was reported (so an agent that says nothing is version 1)
* 2 — reports pending updates, and its own version
* 3 — Linux: the package manager is asked outside the systemd sandbox, so dnf and zypper can answer too (Windows stays at 2)
* 3 — Linux: the package manager is asked outside the systemd sandbox, so dnf and zypper can answer too.
* Windows: can replace itself when an administrator has switched automatic updates on.
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too
*/
import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
@@ -51,6 +54,18 @@ export function latestAgentVersions(dir: string = AGENT_DIR): LatestAgentVersion
return result;
}
/**
* The SHA-256 of the script this app serves for an OS, as hex — what an agent that updates itself checks the download against, so a
* truncated or tampered-with transfer is never installed. Null if the script can't be read.
*/
export function latestAgentSha256(os: AgentOs, dir: string = AGENT_DIR): string | null {
try {
return createHash("sha256").update(readFileSync(join(dir, FILES[os]))).digest("hex");
} catch {
return null;
}
}
/**
* Whether a server's agent is behind. Only a server that reports through an agent can be: one that never has (a Proxmox guest tracked
* through its host, say) has none to update. An agent that reports no version is version 1.
+1
View File
@@ -126,6 +126,7 @@ export function planTicketRun(input: {
const CATEGORY_LABELS: Record<AlertCategory, string> = {
offline: "Server down",
disk: "Disk & storage",
services: "Failing services",
backup: "Backups",
updates: "Updates",
expiry: "Expiry",
+1 -1
View File
@@ -4,7 +4,7 @@
export type AlertSeverity = "critical" | "warning" | "info";
/** What kind of problem — drives the filter on the Alerts page. */
export type AlertCategory = "offline" | "disk" | "backup" | "updates" | "expiry" | "automation" | "integration" | "monitoring" | "tickets";
export type AlertCategory = "offline" | "disk" | "services" | "backup" | "updates" | "expiry" | "automation" | "integration" | "monitoring" | "tickets";
export interface Alert {
/** Stable, so the page can key rows on it. */
+2 -2
View File
@@ -68,8 +68,8 @@ const SEVERITY_ORDER: Record<AlertSeverity, number> = { critical: 0, warning: 1,
/** Which kind of source, and which one, a health/automation condition key belongs to. */
function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } {
let m = /^(?:offline|disk):server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : "disk" };
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : "disk" };
m = /^disk:proxmox:(\d+):/.exec(key);
if (m) return { type: "proxmox", id: Number(m[1]), category: "disk" };
m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key);
+5 -1
View File
@@ -9,6 +9,7 @@ import { recordAudit } from "./audit.js";
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
import { hostHealthConditions, parseHostHealth, type HostHealth } from "./hostHealth.js";
const STATE_FLAG = "healthActiveConditions";
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
@@ -32,6 +33,8 @@ export interface ServerSnapshot {
name: string;
lastSeenAt: string | null;
disks: { mount: string; sizeBytes: number; usedBytes: number }[];
/** What the agent says about failed services, disk health and ZFS pools, if it says anything. */
health?: HostHealth | null;
}
export interface ProxmoxSnapshot {
integrationId: number;
@@ -115,6 +118,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
}
// A server that never reported has no agent to go quiet; and an offline one's disk figures are stale, so neither is judged on disks.
if (s.lastSeenAt && !offline) {
for (const c of hostHealthConditions(s.id, s.name, s.health ?? null)) out.push({ key: c.key, source: "server", message: c.message, severity: c.severity });
for (const d of s.disks) {
const p = usage(d.usedBytes, d.sizeBytes);
if (p !== null && p >= limit) {
@@ -296,7 +300,7 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
} catch {
// a malformed report shouldn't stop every other server being checked
}
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks });
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: parseHostHealth(s.hostHealth) });
}
const rows = await db
+124
View File
@@ -0,0 +1,124 @@
/**
* What a Linux agent can say about the machine's own health, beyond its figures: services that have failed (systemd), whether each disk's
* SMART self-assessment passes and how worn it is, and whether each ZFS pool is online. Each part is optional — an agent reports only what
* it could find out (no smartctl, no ZFS, no systemd) — and a part that is absent from a report keeps what was last said, so an agent that
* skips one never turns an old problem into "all well".
*/
export interface FailedUnit {
unit: string;
description: string;
}
export interface SmartDisk {
device: string;
model: string;
/** The drive's own overall verdict: true passes, false fails, null if it didn't give one. */
healthy: boolean | null;
temperatureC: number | null;
/** NVMe and some SSDs say how much of their rated life is used, 0–100 (and past it). */
wearPercent: number | null;
}
export interface ZfsPool {
name: string;
/** ONLINE, DEGRADED, FAULTED, UNAVAIL, SUSPENDED … as zpool says it. */
health: string;
capacityPercent: number | null;
}
export interface HostHealth {
failedUnits?: FailedUnit[];
smart?: SmartDisk[];
zfs?: ZfsPool[];
/** ISO — when the agent last told us anything here. */
reportedAt: string;
}
/** As it arrives from an agent. */
export interface IncomingHostHealth {
failed_units?: { unit: string; description?: string }[];
smart?: { device: string; model?: string; healthy?: boolean | null; temperature_c?: number | null; wear_percent?: number | null }[];
zfs?: { name: string; health: string; capacity_percent?: number | null }[];
}
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
export function parseHostHealth(stored: string | null): HostHealth | null {
if (!stored) return null;
try {
const v = JSON.parse(stored);
if (!v || typeof v !== "object") return null;
const out: HostHealth = { reportedAt: typeof v.reportedAt === "string" ? v.reportedAt : "" };
if (Array.isArray(v.failedUnits)) out.failedUnits = v.failedUnits.filter((u: any) => u && typeof u.unit === "string").map((u: any) => ({ unit: u.unit, description: typeof u.description === "string" ? u.description : "" }));
if (Array.isArray(v.smart)) {
out.smart = v.smart
.filter((d: any) => d && typeof d.device === "string")
.map((d: any) => ({ device: d.device, model: typeof d.model === "string" ? d.model : "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperatureC), wearPercent: num(d.wearPercent) }));
}
if (Array.isArray(v.zfs)) out.zfs = v.zfs.filter((p: any) => p && typeof p.name === "string" && typeof p.health === "string").map((p: any) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacityPercent) }));
return out;
} catch {
return null;
}
}
/** The text to store after a report: the parts the agent sent replace what was there, the parts it didn't send stay. */
export function nextHostHealth(previous: string | null, incoming: IncomingHostHealth, now: string): string | null {
const keep = parseHostHealth(previous);
const next: HostHealth = { ...(keep ?? {}), reportedAt: now };
let any = false;
if (incoming.failed_units) {
any = true;
next.failedUnits = incoming.failed_units.map((u) => ({ unit: u.unit, description: u.description ?? "" }));
}
if (incoming.smart) {
any = true;
next.smart = incoming.smart.map((d) => ({ device: d.device, model: d.model ?? "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperature_c), wearPercent: num(d.wear_percent) }));
}
if (incoming.zfs) {
any = true;
next.zfs = incoming.zfs.map((p) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacity_percent) }));
}
return any ? JSON.stringify(next) : previous;
}
/** A pool in any state but ONLINE has lost something (DEGRADED has lost redundancy, the rest are worse). */
export const poolIsFine = (p: ZfsPool) => p.health.toUpperCase() === "ONLINE";
/** Wear at which a drive is worth replacing soon, in percent of rated life used. */
export const WEAR_WARNING_PERCENT = 90;
export interface HostHealthCondition {
key: string;
message: string;
severity: "critical" | "warning";
}
const listOf = (names: string[], max = 4) => (names.length <= max ? names.join(", ") : `${names.slice(0, max).join(", ")} and ${names.length - max} more`);
/** The problems a server's health report amounts to. Pure, so the rules can be tested exactly. */
export function hostHealthConditions(serverId: number, serverName: string, health: HostHealth | null): HostHealthCondition[] {
if (!health) return [];
const out: HostHealthCondition[] = [];
const failed = health.failedUnits ?? [];
if (failed.length > 0) {
out.push({
key: `failed-units:server:${serverId}`,
severity: "warning",
message: `${serverName}: ${failed.length === 1 ? "1 service has" : `${failed.length} services have`} failed — ${listOf(failed.map((u) => u.unit))}`,
});
}
for (const d of health.smart ?? []) {
const label = `${d.device}${d.model ? ` (${d.model})` : ""}`;
if (d.healthy === false) {
out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "critical", message: `${serverName}: the disk ${label} says it is failing (SMART self-assessment)` });
} else if (d.wearPercent !== null && d.wearPercent >= WEAR_WARNING_PERCENT) {
out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "warning", message: `${serverName}: the disk ${label} has used ${Math.round(d.wearPercent)}% of its rated life` });
}
}
for (const p of health.zfs ?? []) {
if (!poolIsFine(p)) {
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: ZFS pool ${p.name} is ${p.health}` });
}
}
return out;
}
+1 -1
View File
@@ -23,7 +23,7 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
* so conditions persisted by an earlier version still map correctly.
*/
export function subjectOfConditionKey(key: string): string | null {
let m = /^(?:offline|disk):server:(\d+)(?::|$)/.exec(key);
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)(?::|$)/.exec(key);
if (m) return `server:${m[1]}`;
m = /^disk:proxmox:(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
+20
View File
@@ -10,6 +10,8 @@ import { getAlerts } from "./alerts.js";
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
import { getSourceStatus } from "./integrationHealthMonitor.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { computeSecretStatus } from "./secretStatus.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
@@ -80,8 +82,26 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
if (updates.security !== null) out.add("homelab_server_pending_security_updates", "gauge", "Of those, the ones that are security fixes.", updates.security, { server });
out.add("homelab_server_reboot_required", "gauge", "1 if a restart is due.", updates.rebootRequired ? 1 : 0, { server });
}
const health = parseHostHealth(s.hostHealth);
if (health?.failedUnits) out.add("homelab_server_failed_units", "gauge", "Services (systemd units) that have failed on the server.", health.failedUnits.length, { server });
for (const d of health?.smart ?? []) {
if (d.healthy !== null) out.add("homelab_server_disk_healthy", "gauge", "1 if the disk's own SMART self-assessment passes, 0 if it fails.", d.healthy ? 1 : 0, { server, device: d.device });
if (d.wearPercent !== null) out.add("homelab_server_disk_wear_percent", "gauge", "How much of an SSD's rated life is used.", d.wearPercent, { server, device: d.device });
if (d.temperatureC !== null) out.add("homelab_server_disk_temperature_celsius", "gauge", "A disk's temperature.", d.temperatureC, { server, device: d.device });
}
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a ZFS pool is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
out.add("homelab_server_agent_outdated", "gauge", "1 if the server's agent is older than the one this installation would install now.", isAgentOutdated(s, latest) ? 1 : 0, { server });
}
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
try {
for (const s of serverRows) {
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), 3650, 101)) {
out.add("homelab_server_disk_days_to_full", "gauge", "Days until a disk is full at its present rate of filling; only for disks with a steady climb.", f.daysToFull, { server: s.name, mount: f.mount });
}
}
} catch (err) {
console.error("[metrics] couldn't work out the disk forecasts:", err);
}
try {
const uptime = await getUptime(now, 30);
for (const s of serverRows) {
+2
View File
@@ -54,6 +54,8 @@ export function describeAudit(category: string, action: string, detail: Record<s
return { title: "A disk got nearly full", tone: "bad", kind: "health", detail: message };
case "disk_ok":
return { title: "A disk is no longer nearly full", tone: "good", kind: "health", detail: null };
case "agent_updated":
return { title: `Agent updated to version ${detail.to ?? "?"}`, tone: "good", kind: "change", detail: detail.from !== undefined ? `from version ${detail.from}` : null };
case "create":
return { title: "Server added", tone: "neutral", kind: "change", detail: null };
case "update":
+8
View File
@@ -119,6 +119,12 @@ export interface SignInSettings {
otherwise: "viewer" | "keep";
}
/** How the agents out on the servers are kept up to date. Off until an administrator turns it on. */
export interface AgentSettings {
/** An agent that is behind downloads the newer script from this app, checks it and replaces itself. */
autoUpdate: boolean;
}
/** What the inventory needs to work out a running cost. Without a price per kWh it shows draw but no cost. */
export interface InventorySettings {
/** A short name shown after amounts ("SEK", "kr", "$"); may be blank. */
@@ -180,6 +186,7 @@ export interface AppSettings {
digest: DigestSettings;
signIn: SignInSettings;
inventory: InventorySettings;
agents: AgentSettings;
consistency: ConsistencySettings;
nameGenerator: NameGeneratorSettings;
alertTickets: AlertTicketSettings;
@@ -214,6 +221,7 @@ const DEFAULTS: AppSettings = {
quietHours: { enabled: false, start: "22:00", end: "07:00" },
digest: { enabled: false, day: 1, time: "08:00" },
inventory: { currency: "", pricePerKwh: null },
agents: { autoUpdate: false },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
+15 -1
View File
@@ -3,6 +3,8 @@ import { db } from "../db/client.js";
import { scheduledTasks, servers } from "../db/schema.js";
import { recordServerMetrics } from "./serverMetrics.js";
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
import { nextHostHealth, type IncomingHostHealth } from "./hostHealth.js";
import { recordAudit } from "./audit.js";
export interface IncomingTask {
scheduleType: "cron" | "systemd_timer" | "windows_task";
@@ -23,6 +25,8 @@ export interface IncomingSystemInfo {
listening_ports?: { protocol: "tcp" | "udp"; port: number; address: string; process?: string }[];
/** Updates waiting to be installed. Absent from an agent that doesn't report them (or couldn't find out this time). */
updates?: IncomingUpdates;
/** Failed services, disk health, ZFS pools — each part present only if the agent could find it out. */
health?: IncomingHostHealth;
}
export interface AgentReport {
@@ -103,13 +107,14 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
}
const system = report.system;
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince }).from(servers).where(eq(servers.id, serverId));
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, hostHealth: servers.hostHealth, agentVersion: servers.agentVersion }).from(servers).where(eq(servers.id, serverId));
await db
.update(servers)
.set({
lastSeenAt: now,
// Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before.
...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}),
...(system?.health && before ? { hostHealth: nextHostHealth(before.hostHealth, system.health, now) } : {}),
...(report.hostname ? { hostname: report.hostname } : {}),
...(report.agentVersion !== undefined ? { agentVersion: report.agentVersion } : {}),
// An agent knows what it runs on better than whoever registered the server did.
@@ -132,6 +137,15 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
})
.where(eq(servers.id, serverId));
// An agent that now says a higher version than it did has been updated (by hand, or by itself): worth a line in the server's history.
if (before && report.agentVersion !== undefined && before.agentVersion !== null && report.agentVersion > before.agentVersion) {
try {
await recordAudit({ category: "server", action: "agent_updated", targetType: "server", targetId: serverId, detail: { from: before.agentVersion, to: report.agentVersion } });
} catch (err) {
console.error("[agent] couldn't record the agent update:", err);
}
}
// Keep a little history of the figures too. A failure here must not lose the report itself, which is already saved.
try {
await recordServerMetrics(serverId, system, new Date(now));
+26 -3
View File
@@ -361,7 +361,7 @@ describe("Linux agent", () => {
// The one step that runs outside the sandbox (+) is the package-manager question, which may fail without stopping the report (-),
// and it hands its answer over through the unit's own private directory, which is where the script looks for it.
assert.equal(settings.get("ExecStartPre"), "-+/usr/local/bin/homelab-manager-agent.sh --prepare-updates");
assert.equal(settings.get("ExecStartPre"), "-+/usr/local/bin/homelab-manager-agent.sh --prepare");
assert.equal(rendered.split("\n").filter((l) => /^ExecStartPre=/.test(l.trim())).length, 1, "only that one step is outside the sandbox");
assert.equal(settings.get("RuntimeDirectory"), "homelab-manager-agent");
assert.equal(settings.get("RuntimeDirectoryMode"), "0700");
@@ -399,16 +399,39 @@ describe("Linux agent", () => {
test("the update check can be run on its own, and what it leaves behind is read back through jq, not trusted", () => {
const text = readFileSync(agentScript, "utf8");
assert.ok(text.includes("--prepare-updates) MODE=prepare"));
assert.ok(text.includes("--check-updates) MODE=check"));
assert.ok(text.includes("--prepare|--prepare-updates) MODE=prepare"), "the old name still works, for units written by an earlier installer");
assert.ok(text.includes("--check-updates|--check) MODE=check"));
assert.ok(text.includes("use_prepared_updates || collect_updates"), "the report uses the prepared answer first and asks itself otherwise");
assert.ok(text.includes('find "$UPDATES_FILE" -mmin -10'), "an old answer isn't used");
assert.ok(text.includes("jq -ce 'select(type == \"object\""), "the file is re-shaped by jq, so the report only carries the four known fields");
assert.ok(text.includes('mv -f "$UPDATES_FILE.tmp" "$UPDATES_FILE"'), "written whole, then moved into place");
// disk and pool health need the devices, so they are read from the prepare step and handed over, re-shaped by jq like the updates
assert.ok(text.includes("use_prepared_health"));
assert.ok(text.includes("smartctl -j -H -A -n standby"), "a sleeping disk is not woken to be asked");
assert.ok(text.includes("zpool list -H -o name,health,capacity"));
assert.ok(text.includes("systemctl list-units --state=failed"));
assert.ok(text.includes("pacman -Qu"));
// the two modes must not need a server address or a token
assert.ok(text.includes('[[ "$MODE" == "report" && ( -z "$API_URL" || -z "$API_TOKEN" ) ]]'));
});
test("self-update takes nothing on trust but a checksum, never runs inside the sandbox, and can not loop", () => {
const text = readFileSync(agentScript, "utf8");
const body = text.slice(text.indexOf("self_update() {"), text.indexOf("# The answer --prepare left for this report"));
assert.ok(body.length > 500, "self_update wasn't found");
assert.ok(body.includes("api_get \"/api/agent/report/update?version=$AGENT_VERSION\""), "it asks the server, with its token, whether it should");
assert.ok(body.includes("sha256sum") && body.includes("\"$got\" != \"$sha\""), "the download is checked against the checksum the server gave");
assert.ok(body.includes("bash -n \"$work/new.sh\"") && body.includes("#!/usr/bin/env bash") && body.includes("^AGENT_VERSION=$version"), "and must look like this script, parse, and say the version promised");
assert.ok(body.includes("(( version <= AGENT_VERSION ))"), "only ever to a newer version");
assert.ok(body.includes("mv -f \"$self.new\" \"$self\""), "put beside the old one and moved over it in one step");
assert.ok(body.includes("HLM_JUST_UPDATED"), "the new version doesn't update again straight away");
// it is only called from the prepare step (outside the sandbox) and from the manual check, never from the report
const calls = text.split("\n").filter((l) => /(^|\s|&& )self_update(\s|$)/.test(l) && !l.includes("self_update()") && !l.trim().startsWith("#"));
assert.equal(calls.length, 2, calls.join(" | "));
const reportBegins = text.indexOf("collect_cron\ncollect_systemd_timers");
assert.ok(reportBegins > 0 && text.indexOf("self_update", reportBegins) === -1, "not after the point where a report begins");
});
test("unit files are made readable by everyone (the umask above them is for the token file)", () => {
const text = readFileSync(installScript, "utf8").replace(/\r\n/g, "\n");
assert.match(text, /chmod 644 "\$SYSTEMD_DIR\/homelab-manager-agent\.service" "\$SYSTEMD_DIR\/homelab-manager-agent\.timer"/);
@@ -0,0 +1,316 @@
/**
* What an agent can say about a machine's health (failed services, disk SMART, ZFS pools), the alerts and figures that come from it, and
* the switch that lets agents that are behind replace themselves.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-hosthealth-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, maintenanceWindows } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { hostHealthConditions, nextHostHealth, parseHostHealth } = await import("../src/services/hostHealth.js");
const { evaluateHealth } = await import("../src/services/healthMonitor.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { latestAgentSha256, latestAgentVersions } = await import("../src/services/agentVersions.js");
const { serversRouter } = await import("../src/routes/servers.js");
const { agentReportRouter } = await import("../src/routes/agentReport.js");
const { hashToken } = await import("../src/services/tokens.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const NOW = "2026-10-17T12:00:00.000Z";
const health = (extra: object = {}) => ({ reportedAt: NOW, ...extra });
describe("what a health report amounts to", () => {
test("failed services, a failing disk, a worn one and a pool that isn't online are each a problem; the healthy are not", () => {
const conditions = hostHealthConditions(7, "elsa", {
reportedAt: NOW,
failedUnits: [{ unit: "nginx.service", description: "" }, { unit: "backup.service", description: "" }],
smart: [
{ device: "/dev/sda", model: "WD Red", healthy: false, temperatureC: 40, wearPercent: null },
{ device: "/dev/nvme0", model: "Samsung", healthy: true, temperatureC: 38, wearPercent: 93 },
{ device: "/dev/sdb", model: "Fine", healthy: true, temperatureC: 31, wearPercent: 12 },
{ device: "/dev/sdc", model: "No verdict", healthy: null, temperatureC: null, wearPercent: null },
],
zfs: [{ name: "tank", health: "DEGRADED", capacityPercent: 40 }, { name: "rpool", health: "ONLINE", capacityPercent: 10 }],
});
const byKey = Object.fromEntries(conditions.map((c) => [c.key, c]));
assert.deepEqual(Object.keys(byKey).sort(), ["failed-units:server:7", "smart:server:7:/dev/nvme0", "smart:server:7:/dev/sda", "zfs:server:7:tank"]);
assert.equal(byKey["failed-units:server:7"].severity, "warning");
assert.match(byKey["failed-units:server:7"].message, /2 services have failed — nginx\.service, backup\.service/);
assert.equal(byKey["smart:server:7:/dev/sda"].severity, "critical");
assert.match(byKey["smart:server:7:/dev/sda"].message, /failing/);
assert.equal(byKey["smart:server:7:/dev/nvme0"].severity, "warning");
assert.match(byKey["smart:server:7:/dev/nvme0"].message, /93% of its rated life/);
assert.equal(byKey["zfs:server:7:tank"].severity, "critical");
assert.match(byKey["zfs:server:7:tank"].message, /DEGRADED/);
});
test("nothing said, or everything fine, is no problem; a long list of services is shortened", () => {
assert.deepEqual(hostHealthConditions(1, "x", null), []);
assert.deepEqual(hostHealthConditions(1, "x", health({ failedUnits: [], smart: [], zfs: [] })), []);
const many = Array.from({ length: 9 }, (_, i) => ({ unit: `u${i}.service`, description: "" }));
assert.match(hostHealthConditions(1, "x", health({ failedUnits: many }))[0].message, /and 5 more/);
assert.match(hostHealthConditions(1, "x", health({ failedUnits: [many[0]] }))[0].message, /1 service has failed/);
});
test("a report replaces the parts it carries and keeps the parts it doesn't", () => {
const first = nextHostHealth(null, { failed_units: [{ unit: "a.service" }], zfs: [{ name: "tank", health: "ONLINE", capacity_percent: 20 }] }, "t1");
assert.equal(parseHostHealth(first)!.failedUnits![0].unit, "a.service");
// the next report knows about disks only
const second = nextHostHealth(first, { smart: [{ device: "/dev/sda", healthy: true }] }, "t2");
const merged = parseHostHealth(second)!;
assert.equal(merged.failedUnits![0].unit, "a.service", "services kept");
assert.equal(merged.zfs![0].name, "tank", "pools kept");
assert.equal(merged.smart![0].device, "/dev/sda");
assert.equal(merged.reportedAt, "t2");
// an empty list is an answer ("none failed"), and clears the old one
assert.deepEqual(parseHostHealth(nextHostHealth(second, { failed_units: [] }, "t3"))!.failedUnits, []);
// nothing in the report: what was stored is untouched
assert.equal(nextHostHealth(second, {}, "t4"), second);
assert.equal(parseHostHealth("not json"), null);
});
test("an offline server isn't judged on its health, and an online one is", () => {
const snap = (lastSeenAt: string) => ({ servers: [{ id: 3, name: "elsa", lastSeenAt, disks: [], health: health({ failedUnits: [{ unit: "a.service", description: "" }] }) }], proxmox: [], synology: [] });
const now = Date.parse(NOW);
const limits = { serverOfflineMinutes: 60, diskUsagePercent: 90 };
const online = evaluateHealth(snap(new Date(now - 60_000).toISOString()), limits, now).map((c) => c.key);
assert.deepEqual(online, ["failed-units:server:3"]);
const offline = evaluateHealth(snap(new Date(now - 5 * 3600_000).toISOString()), limits, now).map((c) => c.key);
assert.deepEqual(offline, ["offline:server:3"]);
});
});
describe("through the report, the alerts, the figures and the page", () => {
let app: Server;
let base = "";
let id = 0;
let who = "admin";
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olle", role: "operator" },
]);
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t" }).returning();
id = s.id;
const a = express();
a.use("/api/agent/report", agentReportRouter);
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/servers", serversRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const report = (system: unknown, extra: Record<string, unknown> = {}) =>
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", tasks: [], system, ...extra }) });
const detail = async () => (await (await fetch(`${base}/servers/${id}/detail`)).json()).server;
test("what an agent says is stored and shown; a garbled part costs the report nothing", async () => {
const res = await report({
health: {
failed_units: [{ unit: "nginx.service", description: "A high performance web server" }],
smart: [{ device: "/dev/sda", model: "WD Red", healthy: false, temperature_c: 41, wear_percent: null }],
zfs: "this isn't a list",
},
});
assert.equal(res.status, 202);
let s = await detail();
assert.equal(s.hostHealth.failedUnits[0].unit, "nginx.service");
assert.equal(s.hostHealth.smart[0].healthy, false);
assert.equal(s.hostHealth.zfs, undefined, "the part that made no sense was dropped");
// the list endpoint shows it too, and never as raw stored text
const row = (await (await fetch(`${base}/servers`)).json()).servers.find((x: { id: number }) => x.id === id);
assert.equal(typeof row.hostHealth, "object");
// a report with no health at all leaves it as it was; garbage in place of the whole thing too
assert.equal((await report({ cpu: { cores: 2 } })).status, 202);
assert.equal((await report({ health: 12 })).status, 202);
s = await detail();
assert.equal(s.hostHealth.failedUnits[0].unit, "nginx.service");
});
test("they appear on the Alerts page as their own kind, under the server, and a maintenance window silences them", async () => {
invalidateAlertsCache();
let alerts = (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`));
const services = alerts.find((a) => a.id.startsWith("services:failed-units"));
assert.ok(services, JSON.stringify(alerts.map((a) => a.id)));
assert.equal(services.category, "services");
assert.equal(services.link, `/servers/${id}`);
assert.equal(services.silenced, false);
const disk = alerts.find((a) => a.id.startsWith("disk:smart"));
assert.ok(disk && disk.severity === "critical", "a failing disk is critical, in the disk category");
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: id, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
invalidateAlertsCache();
alerts = (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`));
assert.ok(alerts.length >= 2 && alerts.every((a) => a.silenced), "all held back under maintenance");
await db.delete(maintenanceWindows).where(eq(maintenanceWindows.targetId, id));
// the services come right once the agent says none have failed
await report({ health: { failed_units: [] } });
invalidateAlertsCache();
assert.equal((await getAlerts(true)).alerts.some((a) => a.id.startsWith("services:")), false);
});
test("/metrics carries them", async () => {
await report({ health: { failed_units: [{ unit: "a.service" }, { unit: "b.service" }], zfs: [{ name: "tank", health: "DEGRADED", capacity_percent: 12 }], smart: [{ device: "/dev/sda", healthy: true, temperature_c: 33, wear_percent: 5 }] } });
const text = await buildMetrics();
assert.match(text, /homelab_server_failed_units\{server="elsa"\} 2/);
assert.match(text, /homelab_server_zfs_pool_online\{server="elsa",pool="tank"\} 0/);
assert.match(text, /homelab_server_disk_healthy\{server="elsa",device="\/dev\/sda"\} 1/);
assert.match(text, /homelab_server_disk_wear_percent\{server="elsa",device="\/dev\/sda"\} 5/);
assert.match(text, /homelab_server_disk_temperature_celsius\{server="elsa",device="\/dev\/sda"\} 33/);
});
test("a disk on a steady climb has its days to full in /metrics, and a flat one does not", async () => {
const { serverMetrics } = await import("../src/db/schema.js");
const day = 86_400_000;
const rows = [];
for (let i = 0; i < 10; i++) {
const at = new Date(Date.now() - (10 - i) * day).toISOString();
rows.push({ serverId: id, metric: "disk:/data", value: 50 + i * 2, recordedAt: at });
rows.push({ serverId: id, metric: "disk:/", value: 30, recordedAt: at });
}
await db.insert(serverMetrics).values(rows);
const text = await buildMetrics();
const m = /homelab_server_disk_days_to_full\{server="elsa",mount="\/data"\} (\d+)/.exec(text);
assert.ok(m, text.split("\n").filter((l) => l.includes("days_to_full")).join("\n"));
assert.ok(Number(m[1]) >= 15 && Number(m[1]) <= 17, `about sixteen days (2 points a day, 32 left from 68%), got ${m[1]}`);
assert.doesNotMatch(text, /days_to_full\{server="elsa",mount="\/"\}/);
});
test("an agent that is now a newer version leaves a line in the server's history", async () => {
await report({}, { agent_version: 3 });
await report({}, { agent_version: 4 });
const activity = await (await fetch(`${base}/servers/${id}/activity`)).json();
const line = activity.events.find((e: { title: string }) => /Agent updated to version 4/.test(e.title));
assert.ok(line, JSON.stringify(activity.events.map((e: { title: string }) => e.title)));
assert.equal(line.detail, "from version 3");
assert.equal(activity.events.some((e: { title: string }) => /version 3$/.test(e.title)), false, "the first time a version is seen is not an update");
});
});
describe("agents updating themselves", () => {
let app: Server;
let base = "";
let who = "admin";
let id = 0;
before(async () => {
const [s] = await db.insert(servers).values({ name: "anna", apiTokenHash: hashToken("anna-tok"), apiTokenPrefix: "a", osType: "linux" }).returning();
id = s.id;
const [w] = await db.insert(servers).values({ name: "win", apiTokenHash: hashToken("win-tok"), apiTokenPrefix: "w", osType: "windows", agentVersion: 1 }).returning();
void w;
const a = express();
a.use("/api/agent/report", agentReportRouter);
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/servers", serversRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => app.close());
const ask = (token: string | null, query = "") => fetch(`${base}/agent/report/update${query}`, { headers: token ? { authorization: `Bearer ${token}` } : {} });
const put = (autoUpdate: unknown) => fetch(`${base}/servers/agent-update`, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ autoUpdate }) });
test("only an agent that proves who it is may ask", async () => {
assert.equal((await ask(null)).status, 401);
assert.equal((await ask("nobody")).status, 401);
});
test("while the switch is off nobody is told to update, however far behind", async () => {
const list = await (await fetch(`${base}/servers`)).json();
assert.equal(list.agentAutoUpdate, false, "off until an administrator turns it on");
const reply = await (await ask("anna-tok", "?version=1")).json();
assert.deepEqual(reply, { update: false, version: null, sha256: null });
});
test("only an administrator can switch it, it is written in the audit log, and it shows on the Servers list", async () => {
who = "operator";
assert.equal((await put(true)).status, 403);
who = "admin";
assert.equal((await put("yes")).status, 400);
assert.equal((await put(true)).status, 200);
assert.equal((await (await fetch(`${base}/servers`)).json()).agentAutoUpdate, true);
const { auditLog } = await import("../src/db/schema.js");
const entry = (await db.select().from(auditLog)).find((e) => e.action === "agent_auto_update");
assert.ok(entry);
assert.deepEqual(JSON.parse(entry.detail!), { from: false, to: true });
});
test("with it on, an agent that is behind is told to, with the checksum of what it will fetch; one that is current is not", async () => {
const latest = latestAgentVersions().linux!;
const behind = await (await ask("anna-tok", `?version=${latest - 1}`)).json();
assert.equal(behind.update, true);
assert.equal(behind.version, latest);
assert.equal(behind.sha256, latestAgentSha256("linux"));
assert.match(behind.sha256, /^[0-9a-f]{64}$/);
const current = await (await ask("anna-tok", `?version=${latest}`)).json();
assert.deepEqual(current, { update: false, version: null, sha256: null });
const ahead = await (await ask("anna-tok", `?version=${latest + 5}`)).json();
assert.equal(ahead.update, false);
// no version given: what it last reported counts, and an agent that has said nothing is version 1
assert.equal((await (await ask("anna-tok")).json()).update, true);
await db.update(servers).set({ agentVersion: latest }).where(eq(servers.id, id));
assert.equal((await (await ask("anna-tok")).json()).update, false);
// nonsense is ignored in favour of that
assert.equal((await (await ask("anna-tok", "?version=banana")).json()).update, false);
});
test("each system is given its own script's checksum", async () => {
const reply = await (await ask("win-tok", "?version=1")).json();
assert.equal(reply.update, true);
assert.equal(reply.sha256, latestAgentSha256("windows"));
assert.notEqual(reply.sha256, latestAgentSha256("linux"));
});
test("the checksum is of the file that is served, byte for byte", async () => {
const { createHash } = await import("node:crypto");
const { readFileSync } = await import("node:fs");
const { AGENT_DIR } = await import("../src/services/agentVersions.js");
const bytes = readFileSync(join(AGENT_DIR, "linux", "report-tasks.sh"));
assert.equal(latestAgentSha256("linux"), createHash("sha256").update(bytes).digest("hex"));
assert.equal(latestAgentSha256("linux", join(dir, "empty")), null, "an unreadable script has no checksum, so nobody is told to fetch it");
});
});
+180
View File
@@ -0,0 +1,180 @@
/**
* The Windows agent replacing itself: only with a script the server vouched for (SHA-256), that is newer, says the version promised and
* parses; and only when it is installed under its own name. Run against a fake server, with the real script's own function.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { spawn, spawnSync } from "node:child_process";
import http from "node:http";
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
const realScript = join(repoRoot, "agent", "windows", "report-tasks.ps1");
const probe = process.platform === "win32" ? spawnSync("pwsh", ["-NoProfile", "-Command", "$PSVersionTable.PSVersion.Major"], { encoding: "utf8" }) : null;
const havePwsh = probe?.status === 0;
const sha = (b: Buffer | string) => createHash("sha256").update(b).digest("hex");
/** Runs PowerShell without blocking this process: the fake server these tests talk to lives in it, so a blocking spawn would deadlock. */
function pwshFile(file: string): Promise<{ stdout: string; stderr: string }> {
return new Promise((resolve, reject) => {
const child = spawn("pwsh", ["-NoProfile", "-NonInteractive", "-File", file]);
let stdout = "";
let stderr = "";
const timer = setTimeout(() => child.kill(), 60_000);
child.stdout.on("data", (c) => (stdout += c));
child.stderr.on("data", (c) => (stderr += c));
child.on("error", reject);
child.on("close", () => {
clearTimeout(timer);
resolve({ stdout, stderr });
});
});
}
describe("the Windows agent updating itself", { skip: !havePwsh }, () => {
let work = "";
let fake: http.Server;
let port = 0;
// What the fake server answers; each test sets it.
let reply: { update: boolean; version: number; sha256: string } = { update: false, version: 3, sha256: "" };
let served: Buffer = Buffer.alloc(0);
let asked: string[] = [];
before(async () => {
work = mkdtempSync(join(tmpdir(), "hm-winupdate-"));
fake = http.createServer((req, res) => {
asked.push(`${req.method} ${req.url} auth=${req.headers.authorization ?? "-"}`);
if (req.url?.startsWith("/api/agent/report/update")) {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify(reply));
} else if (req.url === "/agent/windows/report-tasks.ps1") {
res.writeHead(200, { "content-type": "text/plain" });
res.end(served);
} else {
res.writeHead(404).end();
}
});
await new Promise<void>((r) => fake.listen(0, "127.0.0.1", r));
port = (fake.address() as { port: number }).port;
});
after(() => {
fake?.close();
try {
rmSync(work, { recursive: true, force: true });
} catch {
// best effort
}
});
const real = () => readFileSync(realScript);
const OLD = "# the installed agent\n$script:AgentVersion = 1\nWrite-Output 'old'\n";
/** Installs an old stub under `name`, runs the real Update-AgentSelf against the fake server, returns what is left. */
async function attempt(name = "homelab-manager-agent.ps1") {
const dir = mkdtempSync(join(work, "run-"));
mkdirSync(dir, { recursive: true });
const installed = join(dir, name);
writeFileSync(installed, OLD);
asked = [];
const script = `
$ErrorActionPreference = 'Stop'
. '${realScript.replace(/'/g, "''")}'
$script:AgentVersion = 1
$config = @{ ApiUrl = 'http://127.0.0.1:${port}'; ApiToken = 'secret-token'; Insecure = $false }
Update-AgentSelf $config -Self '${installed.replace(/'/g, "''")}'
'DONE'
`;
const scriptFile = join(dir, "run.ps1");
writeFileSync(scriptFile, script);
const r = await pwshFile(scriptFile);
assert.ok(r.stdout.includes("DONE"), `the script did not finish:\n${r.stdout}\n${r.stderr}`);
return { content: readFileSync(installed), leftovers: readdirSync(dir).filter((f) => f.endsWith(".download")), stderr: r.stderr, asked: [...asked] };
}
test("a newer script that matches the checksum replaces the old one, and the token is sent only to the question", async () => {
served = real();
reply = { update: true, version: 3, sha256: sha(served) };
const out = await attempt();
assert.deepEqual(out.content, served, "the installed file is now the served script, byte for byte");
assert.deepEqual(out.leftovers, []);
assert.match(out.stderr, /Updated the agent from version 1 to version 3/);
const question = out.asked.find((l) => l.includes("/api/agent/report/update"))!;
assert.match(question, /version=1/);
assert.match(question, /auth=Bearer secret-token/);
const download = out.asked.find((l) => l.includes("/agent/windows/report-tasks.ps1"))!;
assert.match(download, /auth=-$/, "the public script is fetched without the token");
});
test("nothing happens when the server says no", async () => {
served = real();
reply = { update: false, version: 3, sha256: sha(served) };
const out = await attempt();
assert.equal(out.content.toString(), OLD);
assert.equal(out.asked.some((l) => l.includes("/agent/windows/")), false, "and it does not even download");
});
test("a download that doesn't match the checksum is thrown away", async () => {
served = real();
reply = { update: true, version: 3, sha256: sha("something else") };
const out = await attempt();
assert.equal(out.content.toString(), OLD);
assert.deepEqual(out.leftovers, []);
assert.match(out.stderr, /didn't check out/);
});
test("a script that is not newer, doesn't say the promised version, or doesn't parse is refused even if the checksum fits", async () => {
// not newer than the running version
served = real();
reply = { update: true, version: 1, sha256: sha(served) };
assert.equal((await attempt()).content.toString(), OLD, "version 1 is not newer than 1");
// says a different version than was promised
served = Buffer.from("$script:AgentVersion = 2\nWrite-Output 'x'\n");
reply = { update: true, version: 3, sha256: sha(served) };
assert.equal((await attempt()).content.toString(), OLD);
// says the right version but is not a script
served = Buffer.from("$script:AgentVersion = 3\nfunction Broken { if ( \n");
reply = { update: true, version: 3, sha256: sha(served) };
assert.equal((await attempt()).content.toString(), OLD);
});
test("a copy that isn't installed under the agent's own name is never replaced", async () => {
served = real();
reply = { update: true, version: 3, sha256: sha(served) };
const out = await attempt("report-tasks.ps1");
assert.equal(out.content.toString(), OLD);
assert.equal(out.asked.length, 0, "it does not even ask");
});
test("a server that can't be reached costs nothing", async () => {
const dir = mkdtempSync(join(work, "run-"));
const installed = join(dir, "homelab-manager-agent.ps1");
writeFileSync(installed, OLD);
const script = `
$ErrorActionPreference = 'Stop'
. '${realScript.replace(/'/g, "''")}'
$script:AgentVersion = 1
$config = @{ ApiUrl = 'http://127.0.0.1:1'; ApiToken = 'x'; Insecure = $false }
Update-AgentSelf $config -Self '${installed.replace(/'/g, "''")}'
'DONE'
`;
const scriptFile = join(dir, "run.ps1");
writeFileSync(scriptFile, script);
const r = await pwshFile(scriptFile);
assert.ok(r.stdout.includes("DONE"));
assert.match(r.stderr, /couldn't check for, or install, a newer agent/);
assert.equal(readFileSync(installed, "utf8"), OLD);
});
test("the script stays plain ASCII, as its own header asks", () => {
assert.ok(!/[^\x00-\x7f]/.test(readFileSync(realScript, "utf8")));
});
});