Host health from the Linux agent, agent self-update, and disk forecast in /metrics
- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates; SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox - Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series - Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified download, atomic replace; GET /api/agent/report/update - homelab_server_disk_days_to_full in /metrics - Tests (server, real PowerShell against a fake server) and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
540ab704b7
commit
5dc6cea69c
38 files changed
+3884
-73
No files matched your search
@@ -0,0 +1,124 @@
|
||||
/**
|
||||
* What a Linux agent can say about the machine's own health, beyond its figures: services that have failed (systemd), whether each disk's
|
||||
* SMART self-assessment passes and how worn it is, and whether each ZFS pool is online. Each part is optional — an agent reports only what
|
||||
* it could find out (no smartctl, no ZFS, no systemd) — and a part that is absent from a report keeps what was last said, so an agent that
|
||||
* skips one never turns an old problem into "all well".
|
||||
*/
|
||||
|
||||
export interface FailedUnit {
|
||||
unit: string;
|
||||
description: string;
|
||||
}
|
||||
export interface SmartDisk {
|
||||
device: string;
|
||||
model: string;
|
||||
/** The drive's own overall verdict: true passes, false fails, null if it didn't give one. */
|
||||
healthy: boolean | null;
|
||||
temperatureC: number | null;
|
||||
/** NVMe and some SSDs say how much of their rated life is used, 0–100 (and past it). */
|
||||
wearPercent: number | null;
|
||||
}
|
||||
export interface ZfsPool {
|
||||
name: string;
|
||||
/** ONLINE, DEGRADED, FAULTED, UNAVAIL, SUSPENDED … as zpool says it. */
|
||||
health: string;
|
||||
capacityPercent: number | null;
|
||||
}
|
||||
|
||||
export interface HostHealth {
|
||||
failedUnits?: FailedUnit[];
|
||||
smart?: SmartDisk[];
|
||||
zfs?: ZfsPool[];
|
||||
/** ISO — when the agent last told us anything here. */
|
||||
reportedAt: string;
|
||||
}
|
||||
|
||||
/** As it arrives from an agent. */
|
||||
export interface IncomingHostHealth {
|
||||
failed_units?: { unit: string; description?: string }[];
|
||||
smart?: { device: string; model?: string; healthy?: boolean | null; temperature_c?: number | null; wear_percent?: number | null }[];
|
||||
zfs?: { name: string; health: string; capacity_percent?: number | null }[];
|
||||
}
|
||||
|
||||
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||||
|
||||
export function parseHostHealth(stored: string | null): HostHealth | null {
|
||||
if (!stored) return null;
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
if (!v || typeof v !== "object") return null;
|
||||
const out: HostHealth = { reportedAt: typeof v.reportedAt === "string" ? v.reportedAt : "" };
|
||||
if (Array.isArray(v.failedUnits)) out.failedUnits = v.failedUnits.filter((u: any) => u && typeof u.unit === "string").map((u: any) => ({ unit: u.unit, description: typeof u.description === "string" ? u.description : "" }));
|
||||
if (Array.isArray(v.smart)) {
|
||||
out.smart = v.smart
|
||||
.filter((d: any) => d && typeof d.device === "string")
|
||||
.map((d: any) => ({ device: d.device, model: typeof d.model === "string" ? d.model : "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperatureC), wearPercent: num(d.wearPercent) }));
|
||||
}
|
||||
if (Array.isArray(v.zfs)) out.zfs = v.zfs.filter((p: any) => p && typeof p.name === "string" && typeof p.health === "string").map((p: any) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacityPercent) }));
|
||||
return out;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The text to store after a report: the parts the agent sent replace what was there, the parts it didn't send stay. */
|
||||
export function nextHostHealth(previous: string | null, incoming: IncomingHostHealth, now: string): string | null {
|
||||
const keep = parseHostHealth(previous);
|
||||
const next: HostHealth = { ...(keep ?? {}), reportedAt: now };
|
||||
let any = false;
|
||||
if (incoming.failed_units) {
|
||||
any = true;
|
||||
next.failedUnits = incoming.failed_units.map((u) => ({ unit: u.unit, description: u.description ?? "" }));
|
||||
}
|
||||
if (incoming.smart) {
|
||||
any = true;
|
||||
next.smart = incoming.smart.map((d) => ({ device: d.device, model: d.model ?? "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperature_c), wearPercent: num(d.wear_percent) }));
|
||||
}
|
||||
if (incoming.zfs) {
|
||||
any = true;
|
||||
next.zfs = incoming.zfs.map((p) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacity_percent) }));
|
||||
}
|
||||
return any ? JSON.stringify(next) : previous;
|
||||
}
|
||||
|
||||
/** A pool in any state but ONLINE has lost something (DEGRADED has lost redundancy, the rest are worse). */
|
||||
export const poolIsFine = (p: ZfsPool) => p.health.toUpperCase() === "ONLINE";
|
||||
|
||||
/** Wear at which a drive is worth replacing soon, in percent of rated life used. */
|
||||
export const WEAR_WARNING_PERCENT = 90;
|
||||
|
||||
export interface HostHealthCondition {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "critical" | "warning";
|
||||
}
|
||||
|
||||
const listOf = (names: string[], max = 4) => (names.length <= max ? names.join(", ") : `${names.slice(0, max).join(", ")} and ${names.length - max} more`);
|
||||
|
||||
/** The problems a server's health report amounts to. Pure, so the rules can be tested exactly. */
|
||||
export function hostHealthConditions(serverId: number, serverName: string, health: HostHealth | null): HostHealthCondition[] {
|
||||
if (!health) return [];
|
||||
const out: HostHealthCondition[] = [];
|
||||
const failed = health.failedUnits ?? [];
|
||||
if (failed.length > 0) {
|
||||
out.push({
|
||||
key: `failed-units:server:${serverId}`,
|
||||
severity: "warning",
|
||||
message: `${serverName}: ${failed.length === 1 ? "1 service has" : `${failed.length} services have`} failed — ${listOf(failed.map((u) => u.unit))}`,
|
||||
});
|
||||
}
|
||||
for (const d of health.smart ?? []) {
|
||||
const label = `${d.device}${d.model ? ` (${d.model})` : ""}`;
|
||||
if (d.healthy === false) {
|
||||
out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "critical", message: `${serverName}: the disk ${label} says it is failing (SMART self-assessment)` });
|
||||
} else if (d.wearPercent !== null && d.wearPercent >= WEAR_WARNING_PERCENT) {
|
||||
out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "warning", message: `${serverName}: the disk ${label} has used ${Math.round(d.wearPercent)}% of its rated life` });
|
||||
}
|
||||
}
|
||||
for (const p of health.zfs ?? []) {
|
||||
if (!poolIsFine(p)) {
|
||||
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: ZFS pool ${p.name} is ${p.health}` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
Reference in new issue
Block a user