/** * What a Linux agent can say about the machine's own health, beyond its figures: services that have failed (systemd), whether each disk's * SMART self-assessment passes and how worn it is, and whether each ZFS pool is online. Each part is optional — an agent reports only what * it could find out (no smartctl, no ZFS, no systemd) — and a part that is absent from a report keeps what was last said, so an agent that * skips one never turns an old problem into "all well". */ export interface FailedUnit { unit: string; description: string; } export interface SmartDisk { device: string; model: string; /** The drive's own overall verdict: true passes, false fails, null if it didn't give one. */ healthy: boolean | null; temperatureC: number | null; /** NVMe and some SSDs say how much of their rated life is used, 0–100 (and past it). */ wearPercent: number | null; } /** A storage pool: ZFS on Linux, Storage Spaces on Windows (which reports Healthy, Warning and Unhealthy as ONLINE, DEGRADED and FAULTED). */ export interface ZfsPool { name: string; /** ONLINE, DEGRADED, FAULTED, UNAVAIL, SUSPENDED … as zpool says it. */ health: string; capacityPercent: number | null; } export interface HostHealth { failedUnits?: FailedUnit[]; smart?: SmartDisk[]; zfs?: ZfsPool[]; /** ISO — when the agent last told us anything here. */ reportedAt: string; } /** As it arrives from an agent. */ export interface IncomingHostHealth { failed_units?: { unit: string; description?: string }[]; smart?: { device: string; model?: string; healthy?: boolean | null; temperature_c?: number | null; wear_percent?: number | null }[]; zfs?: { name: string; health: string; capacity_percent?: number | null }[]; } const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null); export function parseHostHealth(stored: string | null): HostHealth | null { if (!stored) return null; try { const v = JSON.parse(stored); if (!v || typeof v !== "object") return null; const out: HostHealth = { reportedAt: typeof v.reportedAt === "string" ? v.reportedAt : "" }; if (Array.isArray(v.failedUnits)) out.failedUnits = v.failedUnits.filter((u: any) => u && typeof u.unit === "string").map((u: any) => ({ unit: u.unit, description: typeof u.description === "string" ? u.description : "" })); if (Array.isArray(v.smart)) { out.smart = v.smart .filter((d: any) => d && typeof d.device === "string") .map((d: any) => ({ device: d.device, model: typeof d.model === "string" ? d.model : "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperatureC), wearPercent: num(d.wearPercent) })); } if (Array.isArray(v.zfs)) out.zfs = v.zfs.filter((p: any) => p && typeof p.name === "string" && typeof p.health === "string").map((p: any) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacityPercent) })); return out; } catch { return null; } } /** The text to store after a report: the parts the agent sent replace what was there, the parts it didn't send stay. */ export function nextHostHealth(previous: string | null, incoming: IncomingHostHealth, now: string): string | null { const keep = parseHostHealth(previous); const next: HostHealth = { ...(keep ?? {}), reportedAt: now }; let any = false; if (incoming.failed_units) { any = true; next.failedUnits = incoming.failed_units.map((u) => ({ unit: u.unit, description: u.description ?? "" })); } if (incoming.smart) { any = true; next.smart = incoming.smart.map((d) => ({ device: d.device, model: d.model ?? "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperature_c), wearPercent: num(d.wear_percent) })); } if (incoming.zfs) { any = true; next.zfs = incoming.zfs.map((p) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacity_percent) })); } return any ? JSON.stringify(next) : previous; } /** A pool in any state but ONLINE has lost something (DEGRADED has lost redundancy, the rest are worse). */ export const poolIsFine = (p: ZfsPool) => p.health.toUpperCase() === "ONLINE"; /** Wear at which a drive is worth replacing soon, in percent of rated life used. */ export const WEAR_WARNING_PERCENT = 90; export interface HostHealthCondition { key: string; message: string; severity: "critical" | "warning"; } const listOf = (names: string[], max = 4) => (names.length <= max ? names.join(", ") : `${names.slice(0, max).join(", ")} and ${names.length - max} more`); /** The problems a server's health report amounts to. Pure, so the rules can be tested exactly. */ export function hostHealthConditions(serverId: number, serverName: string, health: HostHealth | null): HostHealthCondition[] { if (!health) return []; const out: HostHealthCondition[] = []; const failed = health.failedUnits ?? []; if (failed.length > 0) { out.push({ key: `failed-units:server:${serverId}`, severity: "warning", message: `${serverName}: ${failed.length === 1 ? "1 service has" : `${failed.length} services have`} failed — ${listOf(failed.map((u) => u.unit))}`, }); } for (const d of health.smart ?? []) { const label = `${d.device}${d.model ? ` (${d.model})` : ""}`; if (d.healthy === false) { out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "critical", message: `${serverName}: the disk ${label} says it is failing (SMART self-assessment)` }); } else if (d.wearPercent !== null && d.wearPercent >= WEAR_WARNING_PERCENT) { out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "warning", message: `${serverName}: the disk ${label} has used ${Math.round(d.wearPercent)}% of its rated life` }); } } for (const p of health.zfs ?? []) { if (!poolIsFine(p)) { out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: storage pool ${p.name} is ${p.health}` }); } } return out; }