- Dependencies: servers and integrations can depend on each other; alerts are folded under the root cause, and a failure explained by something down isn't announced - Web push: hand-written VAPID and RFC 8291 encryption, per-device subscriptions, a fifth notification channel, service worker handlers - Inventory: maker/model/serial from the agents fill blank fields, "Refresh from agent", QR label printing and ?asset= deep links - Windows agent (v4) reports failed services, disk health, Storage Spaces pools and hardware identity; Linux agent (v5) reports hardware identity Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
126 lines
5.9 KiB
TypeScript
126 lines
5.9 KiB
TypeScript
/**
|
||
* What a Linux agent can say about the machine's own health, beyond its figures: services that have failed (systemd), whether each disk's
|
||
* SMART self-assessment passes and how worn it is, and whether each ZFS pool is online. Each part is optional — an agent reports only what
|
||
* it could find out (no smartctl, no ZFS, no systemd) — and a part that is absent from a report keeps what was last said, so an agent that
|
||
* skips one never turns an old problem into "all well".
|
||
*/
|
||
|
||
export interface FailedUnit {
|
||
unit: string;
|
||
description: string;
|
||
}
|
||
export interface SmartDisk {
|
||
device: string;
|
||
model: string;
|
||
/** The drive's own overall verdict: true passes, false fails, null if it didn't give one. */
|
||
healthy: boolean | null;
|
||
temperatureC: number | null;
|
||
/** NVMe and some SSDs say how much of their rated life is used, 0–100 (and past it). */
|
||
wearPercent: number | null;
|
||
}
|
||
/** A storage pool: ZFS on Linux, Storage Spaces on Windows (which reports Healthy, Warning and Unhealthy as ONLINE, DEGRADED and FAULTED). */
|
||
export interface ZfsPool {
|
||
name: string;
|
||
/** ONLINE, DEGRADED, FAULTED, UNAVAIL, SUSPENDED … as zpool says it. */
|
||
health: string;
|
||
capacityPercent: number | null;
|
||
}
|
||
|
||
export interface HostHealth {
|
||
failedUnits?: FailedUnit[];
|
||
smart?: SmartDisk[];
|
||
zfs?: ZfsPool[];
|
||
/** ISO — when the agent last told us anything here. */
|
||
reportedAt: string;
|
||
}
|
||
|
||
/** As it arrives from an agent. */
|
||
export interface IncomingHostHealth {
|
||
failed_units?: { unit: string; description?: string }[];
|
||
smart?: { device: string; model?: string; healthy?: boolean | null; temperature_c?: number | null; wear_percent?: number | null }[];
|
||
zfs?: { name: string; health: string; capacity_percent?: number | null }[];
|
||
}
|
||
|
||
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||
|
||
export function parseHostHealth(stored: string | null): HostHealth | null {
|
||
if (!stored) return null;
|
||
try {
|
||
const v = JSON.parse(stored);
|
||
if (!v || typeof v !== "object") return null;
|
||
const out: HostHealth = { reportedAt: typeof v.reportedAt === "string" ? v.reportedAt : "" };
|
||
if (Array.isArray(v.failedUnits)) out.failedUnits = v.failedUnits.filter((u: any) => u && typeof u.unit === "string").map((u: any) => ({ unit: u.unit, description: typeof u.description === "string" ? u.description : "" }));
|
||
if (Array.isArray(v.smart)) {
|
||
out.smart = v.smart
|
||
.filter((d: any) => d && typeof d.device === "string")
|
||
.map((d: any) => ({ device: d.device, model: typeof d.model === "string" ? d.model : "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperatureC), wearPercent: num(d.wearPercent) }));
|
||
}
|
||
if (Array.isArray(v.zfs)) out.zfs = v.zfs.filter((p: any) => p && typeof p.name === "string" && typeof p.health === "string").map((p: any) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacityPercent) }));
|
||
return out;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/** The text to store after a report: the parts the agent sent replace what was there, the parts it didn't send stay. */
|
||
export function nextHostHealth(previous: string | null, incoming: IncomingHostHealth, now: string): string | null {
|
||
const keep = parseHostHealth(previous);
|
||
const next: HostHealth = { ...(keep ?? {}), reportedAt: now };
|
||
let any = false;
|
||
if (incoming.failed_units) {
|
||
any = true;
|
||
next.failedUnits = incoming.failed_units.map((u) => ({ unit: u.unit, description: u.description ?? "" }));
|
||
}
|
||
if (incoming.smart) {
|
||
any = true;
|
||
next.smart = incoming.smart.map((d) => ({ device: d.device, model: d.model ?? "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperature_c), wearPercent: num(d.wear_percent) }));
|
||
}
|
||
if (incoming.zfs) {
|
||
any = true;
|
||
next.zfs = incoming.zfs.map((p) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacity_percent) }));
|
||
}
|
||
return any ? JSON.stringify(next) : previous;
|
||
}
|
||
|
||
/** A pool in any state but ONLINE has lost something (DEGRADED has lost redundancy, the rest are worse). */
|
||
export const poolIsFine = (p: ZfsPool) => p.health.toUpperCase() === "ONLINE";
|
||
|
||
/** Wear at which a drive is worth replacing soon, in percent of rated life used. */
|
||
export const WEAR_WARNING_PERCENT = 90;
|
||
|
||
export interface HostHealthCondition {
|
||
key: string;
|
||
message: string;
|
||
severity: "critical" | "warning";
|
||
}
|
||
|
||
const listOf = (names: string[], max = 4) => (names.length <= max ? names.join(", ") : `${names.slice(0, max).join(", ")} and ${names.length - max} more`);
|
||
|
||
/** The problems a server's health report amounts to. Pure, so the rules can be tested exactly. */
|
||
export function hostHealthConditions(serverId: number, serverName: string, health: HostHealth | null): HostHealthCondition[] {
|
||
if (!health) return [];
|
||
const out: HostHealthCondition[] = [];
|
||
const failed = health.failedUnits ?? [];
|
||
if (failed.length > 0) {
|
||
out.push({
|
||
key: `failed-units:server:${serverId}`,
|
||
severity: "warning",
|
||
message: `${serverName}: ${failed.length === 1 ? "1 service has" : `${failed.length} services have`} failed — ${listOf(failed.map((u) => u.unit))}`,
|
||
});
|
||
}
|
||
for (const d of health.smart ?? []) {
|
||
const label = `${d.device}${d.model ? ` (${d.model})` : ""}`;
|
||
if (d.healthy === false) {
|
||
out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "critical", message: `${serverName}: the disk ${label} says it is failing (SMART self-assessment)` });
|
||
} else if (d.wearPercent !== null && d.wearPercent >= WEAR_WARNING_PERCENT) {
|
||
out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "warning", message: `${serverName}: the disk ${label} has used ${Math.round(d.wearPercent)}% of its rated life` });
|
||
}
|
||
}
|
||
for (const p of health.zfs ?? []) {
|
||
if (!poolIsFine(p)) {
|
||
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: storage pool ${p.name} is ${p.health}` });
|
||
}
|
||
}
|
||
return out;
|
||
}
|