Files
Homelab-manager/server/src/services/agentDiagnostics.ts
T
bobbanandClaude Sonnet 5.5 c3bc6bbf72 Agent diagnostics ("what the agent couldn't collect") and services to ignore per server
- Agents (Linux 6, Windows 5) note each check they couldn't make and why; the server page turns that into tips and fixes, including when the agent is behind
- Linux --prepare leaves its notes for the report; a missing prepare step is itself noted
- Services to ignore are set per server from the Host health card (names or * patterns), applied on the server so alerts and /metrics drop them at once; saving alert limits keeps the list

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-18 22:16:31 +02:00

131 lines
7.7 KiB
TypeScript

/**
* What an agent could not collect, and what to do about it. Agents (Linux version 6, Windows version 5) note each check they couldn't make —
* with whether nothing was there to read it with ("unavailable") or it tried and couldn't ("failed"), why in a word and at most a line of
* what the tool said. They say only problems; a check not mentioned went fine. The words and the fixes live here, not in the agents, so they
* can be improved without installing anything again.
*/
import { isAgentOutdated, type LatestAgentVersions } from "./agentVersions.js";
export interface AgentDiagnostic {
check: string;
status: "unavailable" | "failed";
reason?: string;
detail?: string;
}
export interface StoredDiagnostics {
items: AgentDiagnostic[];
/** ISO — the report these came from. */
reportedAt: string;
}
export interface DiagnosticNote {
check: string;
/** "warning": something to look at. "info": a tip, or something that is simply how this machine is. */
level: "warning" | "info";
title: string;
fix: string | null;
/** What the agent said about it, when it said anything. */
detail: string | null;
}
const clean = (v: unknown, max: number) => (typeof v === "string" ? v.replace(/[\u0000-\u001f]/g, " ").trim().slice(0, max) : "");
/** Reads what is stored, leniently. Null when the agent has never said (an agent that predates this, or one that has nothing to report on). */
export function parseDiagnostics(stored: string | null | undefined): StoredDiagnostics | null {
if (!stored) return null;
try {
const v = JSON.parse(stored);
if (!v || typeof v !== "object" || !Array.isArray(v.items)) return null;
const items: AgentDiagnostic[] = [];
for (const i of v.items.slice(0, 20)) {
if (!i || typeof i !== "object") continue;
const check = clean(i.check, 40);
if (!check || (i.status !== "unavailable" && i.status !== "failed")) continue;
const item: AgentDiagnostic = { check, status: i.status };
const reason = clean(i.reason, 40);
const detail = clean(i.detail, 160);
if (reason) item.reason = reason;
if (detail) item.detail = detail;
items.push(item);
}
return { items, reportedAt: typeof v.reportedAt === "string" ? v.reportedAt : "" };
} catch {
return null;
}
}
/** What to store after a report that carried notes (an empty list is an answer: nothing was missed). */
export function nextDiagnostics(incoming: AgentDiagnostic[], now: string): string {
return JSON.stringify({ items: incoming, reportedAt: now } satisfies StoredDiagnostics);
}
type Os = "linux" | "windows";
/** One note for what the agent said, or null when it is not worth a person's attention (no ZFS on a machine that has none). */
function describe(d: AgentDiagnostic, os: Os): Omit<DiagnosticNote, "detail"> | null {
const linux = os === "linux";
const key = `${d.check}/${d.status}${d.reason ? `/${d.reason}` : ""}`;
switch (key) {
case "prepare/unavailable/step_missing":
return {
check: d.check,
level: "warning",
title: "The agent's step outside its sandbox didn't run",
fix: "Disk health and ZFS pools are only read in that step. Install the agent again (Servers → Manage servers has the command) so its systemd unit has it; if it is installed, look at `journalctl -u homelab-manager-agent` on the host.",
};
case "updates/unavailable/no_package_manager":
return { check: d.check, level: "info", title: "Pending updates can't be read here", fix: "The agent understands apt, dnf, zypper and pacman, and Windows Update. This machine has none of them." };
case "failed_units/unavailable/no_systemd":
return { check: d.check, level: "info", title: "Failed services can't be read: this machine doesn't run systemd", fix: null };
case "smart/unavailable/tool_missing":
return { check: d.check, level: "info", title: "Disk health isn't reported: smartctl isn't installed", fix: "Install smartmontools on the host (apt install smartmontools, or dnf install smartmontools); the next report includes it." };
case "smart/unavailable/not_root":
return { check: d.check, level: "warning", title: "Disk health needs the agent to run as root", fix: "The install script's systemd unit runs it as root. A copy started by hand as another user can't read the disks." };
case "zfs/unavailable/tool_missing":
return null; // most machines have no ZFS, and that is not a problem
case "hardware/unavailable/not_found":
return { check: d.check, level: "info", title: "No maker, model or serial number found", fix: "Many virtual machines and single-board computers don't have them. Type them into the Inventory instead." };
}
if (d.status === "failed") {
const what: Record<string, string> = { updates: "pending updates", failed_units: "services", smart: "disk health", zfs: "storage pools", host_health: "the machine's health", hardware: "what the machine is" };
const fixes: Record<string, string> = linux
? {
updates: "Run `sudo /usr/local/bin/homelab-manager-agent.sh --check-updates` on the host to see what the package manager said (dnf and zypper answer from their caches: `sudo dnf makecache`).",
failed_units: "`systemctl list-units --state=failed` didn't answer in time on that host.",
zfs: "`zpool list` didn't answer; check that the ZFS kernel module is loaded.",
}
: {
updates: "Windows Update didn't answer (the search is tried again after an hour). Check the Windows Update service and that the machine can reach Microsoft.",
failed_units: "Reading services through WMI failed. Check that the Windows Management Instrumentation service is running.",
smart: "Get-PhysicalDisk failed. In an elevated PowerShell, run `Get-PhysicalDisk` to see why.",
zfs: "Get-StoragePool failed. In an elevated PowerShell, run `Get-StoragePool` to see why.",
host_health: "Reading the machine's health failed; the detail says why.",
hardware: "Reading the system's own details through WMI failed; the detail says why.",
};
return { check: d.check, level: "warning", title: `Reading ${what[d.check] ?? d.check} failed`, fix: fixes[d.check] ?? null };
}
return { check: d.check, level: "info", title: `${d.check.replace(/_/g, " ")} isn't available${d.reason ? ` (${d.reason.replace(/_/g, " ")})` : ""}`, fix: null };
}
/** What to tell a person about one server's agent: what it couldn't collect (with what to do) and, when it is behind, that. Warnings first. */
export function describeDiagnostics(stored: StoredDiagnostics | null, context: { osType: string; lastSeenAt: string | null; agentVersion: number | null; latest: LatestAgentVersions }): DiagnosticNote[] {
const os: Os = context.osType === "windows" ? "windows" : "linux";
const notes: DiagnosticNote[] = [];
for (const d of stored?.items ?? []) {
const note = describe(d, os);
if (note) notes.push({ ...note, detail: d.status === "failed" ? (d.detail ?? null) : null }); // what the agent said is only worth showing for a failure
}
if (isAgentOutdated({ osType: context.osType, lastSeenAt: context.lastSeenAt, agentVersion: context.agentVersion }, context.latest)) {
const current = os === "windows" ? context.latest.windows : context.latest.linux;
notes.push({
check: "agent_behind",
level: "info",
title: `This agent is version ${context.agentVersion ?? 1}; ${current} is current`,
fix: "A newer agent reports more and says more about what it couldn't read. Update it from the Updates page, or switch on automatic updates for agents in Settings.",
detail: null,
});
}
return notes.sort((a, b) => Number(b.level === "warning") - Number(a.level === "warning"));
}