Host health from the Linux agent, agent self-update, and disk forecast in /metrics

- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates;
  SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox
- Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new
  "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series
- Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified
  download, atomic replace; GET /api/agent/report/update
- homelab_server_disk_days_to_full in /metrics
- Tests (server, real PowerShell against a fake server) and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 00:54:24 +02:00
1 parent 540ab704b7
commit 5dc6cea69c
38 files changed
+3884 -73

No files matched your search

+124
View File
@@ -0,0 +1,124 @@
/**
* What a Linux agent can say about the machine's own health, beyond its figures: services that have failed (systemd), whether each disk's
* SMART self-assessment passes and how worn it is, and whether each ZFS pool is online. Each part is optional — an agent reports only what
* it could find out (no smartctl, no ZFS, no systemd) — and a part that is absent from a report keeps what was last said, so an agent that
* skips one never turns an old problem into "all well".
*/
export interface FailedUnit {
unit: string;
description: string;
}
export interface SmartDisk {
device: string;
model: string;
/** The drive's own overall verdict: true passes, false fails, null if it didn't give one. */
healthy: boolean | null;
temperatureC: number | null;
/** NVMe and some SSDs say how much of their rated life is used, 0–100 (and past it). */
wearPercent: number | null;
}
export interface ZfsPool {
name: string;
/** ONLINE, DEGRADED, FAULTED, UNAVAIL, SUSPENDED … as zpool says it. */
health: string;
capacityPercent: number | null;
}
export interface HostHealth {
failedUnits?: FailedUnit[];
smart?: SmartDisk[];
zfs?: ZfsPool[];
/** ISO — when the agent last told us anything here. */
reportedAt: string;
}
/** As it arrives from an agent. */
export interface IncomingHostHealth {
failed_units?: { unit: string; description?: string }[];
smart?: { device: string; model?: string; healthy?: boolean | null; temperature_c?: number | null; wear_percent?: number | null }[];
zfs?: { name: string; health: string; capacity_percent?: number | null }[];
}
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
export function parseHostHealth(stored: string | null): HostHealth | null {
if (!stored) return null;
try {
const v = JSON.parse(stored);
if (!v || typeof v !== "object") return null;
const out: HostHealth = { reportedAt: typeof v.reportedAt === "string" ? v.reportedAt : "" };
if (Array.isArray(v.failedUnits)) out.failedUnits = v.failedUnits.filter((u: any) => u && typeof u.unit === "string").map((u: any) => ({ unit: u.unit, description: typeof u.description === "string" ? u.description : "" }));
if (Array.isArray(v.smart)) {
out.smart = v.smart
.filter((d: any) => d && typeof d.device === "string")
.map((d: any) => ({ device: d.device, model: typeof d.model === "string" ? d.model : "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperatureC), wearPercent: num(d.wearPercent) }));
}
if (Array.isArray(v.zfs)) out.zfs = v.zfs.filter((p: any) => p && typeof p.name === "string" && typeof p.health === "string").map((p: any) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacityPercent) }));
return out;
} catch {
return null;
}
}
/** The text to store after a report: the parts the agent sent replace what was there, the parts it didn't send stay. */
export function nextHostHealth(previous: string | null, incoming: IncomingHostHealth, now: string): string | null {
const keep = parseHostHealth(previous);
const next: HostHealth = { ...(keep ?? {}), reportedAt: now };
let any = false;
if (incoming.failed_units) {
any = true;
next.failedUnits = incoming.failed_units.map((u) => ({ unit: u.unit, description: u.description ?? "" }));
}
if (incoming.smart) {
any = true;
next.smart = incoming.smart.map((d) => ({ device: d.device, model: d.model ?? "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperature_c), wearPercent: num(d.wear_percent) }));
}
if (incoming.zfs) {
any = true;
next.zfs = incoming.zfs.map((p) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacity_percent) }));
}
return any ? JSON.stringify(next) : previous;
}
/** A pool in any state but ONLINE has lost something (DEGRADED has lost redundancy, the rest are worse). */
export const poolIsFine = (p: ZfsPool) => p.health.toUpperCase() === "ONLINE";
/** Wear at which a drive is worth replacing soon, in percent of rated life used. */
export const WEAR_WARNING_PERCENT = 90;
export interface HostHealthCondition {
key: string;
message: string;
severity: "critical" | "warning";
}
const listOf = (names: string[], max = 4) => (names.length <= max ? names.join(", ") : `${names.slice(0, max).join(", ")} and ${names.length - max} more`);
/** The problems a server's health report amounts to. Pure, so the rules can be tested exactly. */
export function hostHealthConditions(serverId: number, serverName: string, health: HostHealth | null): HostHealthCondition[] {
if (!health) return [];
const out: HostHealthCondition[] = [];
const failed = health.failedUnits ?? [];
if (failed.length > 0) {
out.push({
key: `failed-units:server:${serverId}`,
severity: "warning",
message: `${serverName}: ${failed.length === 1 ? "1 service has" : `${failed.length} services have`} failed — ${listOf(failed.map((u) => u.unit))}`,
});
}
for (const d of health.smart ?? []) {
const label = `${d.device}${d.model ? ` (${d.model})` : ""}`;
if (d.healthy === false) {
out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "critical", message: `${serverName}: the disk ${label} says it is failing (SMART self-assessment)` });
} else if (d.wearPercent !== null && d.wearPercent >= WEAR_WARNING_PERCENT) {
out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "warning", message: `${serverName}: the disk ${label} has used ${Math.round(d.wearPercent)}% of its rated life` });
}
}
for (const p of health.zfs ?? []) {
if (!poolIsFine(p)) {
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: ZFS pool ${p.name} is ${p.health}` });
}
}
return out;
}