- Agents (Linux 6, Windows 5) note each check they couldn't make and why; the server page turns that into tips and fixes, including when the agent is behind - Linux --prepare leaves its notes for the report; a missing prepare step is itself noted - Services to ignore are set per server from the Host health card (names or * patterns), applied on the server so alerts and /metrics drop them at once; saving alert limits keeps the list Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
131 lines
7.7 KiB
TypeScript
131 lines
7.7 KiB
TypeScript
/**
|
|
* What an agent could not collect, and what to do about it. Agents (Linux version 6, Windows version 5) note each check they couldn't make —
|
|
* with whether nothing was there to read it with ("unavailable") or it tried and couldn't ("failed"), why in a word and at most a line of
|
|
* what the tool said. They say only problems; a check not mentioned went fine. The words and the fixes live here, not in the agents, so they
|
|
* can be improved without installing anything again.
|
|
*/
|
|
import { isAgentOutdated, type LatestAgentVersions } from "./agentVersions.js";
|
|
|
|
export interface AgentDiagnostic {
|
|
check: string;
|
|
status: "unavailable" | "failed";
|
|
reason?: string;
|
|
detail?: string;
|
|
}
|
|
|
|
export interface StoredDiagnostics {
|
|
items: AgentDiagnostic[];
|
|
/** ISO — the report these came from. */
|
|
reportedAt: string;
|
|
}
|
|
|
|
export interface DiagnosticNote {
|
|
check: string;
|
|
/** "warning": something to look at. "info": a tip, or something that is simply how this machine is. */
|
|
level: "warning" | "info";
|
|
title: string;
|
|
fix: string | null;
|
|
/** What the agent said about it, when it said anything. */
|
|
detail: string | null;
|
|
}
|
|
|
|
const clean = (v: unknown, max: number) => (typeof v === "string" ? v.replace(/[\u0000-\u001f]/g, " ").trim().slice(0, max) : "");
|
|
|
|
/** Reads what is stored, leniently. Null when the agent has never said (an agent that predates this, or one that has nothing to report on). */
|
|
export function parseDiagnostics(stored: string | null | undefined): StoredDiagnostics | null {
|
|
if (!stored) return null;
|
|
try {
|
|
const v = JSON.parse(stored);
|
|
if (!v || typeof v !== "object" || !Array.isArray(v.items)) return null;
|
|
const items: AgentDiagnostic[] = [];
|
|
for (const i of v.items.slice(0, 20)) {
|
|
if (!i || typeof i !== "object") continue;
|
|
const check = clean(i.check, 40);
|
|
if (!check || (i.status !== "unavailable" && i.status !== "failed")) continue;
|
|
const item: AgentDiagnostic = { check, status: i.status };
|
|
const reason = clean(i.reason, 40);
|
|
const detail = clean(i.detail, 160);
|
|
if (reason) item.reason = reason;
|
|
if (detail) item.detail = detail;
|
|
items.push(item);
|
|
}
|
|
return { items, reportedAt: typeof v.reportedAt === "string" ? v.reportedAt : "" };
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** What to store after a report that carried notes (an empty list is an answer: nothing was missed). */
|
|
export function nextDiagnostics(incoming: AgentDiagnostic[], now: string): string {
|
|
return JSON.stringify({ items: incoming, reportedAt: now } satisfies StoredDiagnostics);
|
|
}
|
|
|
|
type Os = "linux" | "windows";
|
|
|
|
/** One note for what the agent said, or null when it is not worth a person's attention (no ZFS on a machine that has none). */
|
|
function describe(d: AgentDiagnostic, os: Os): Omit<DiagnosticNote, "detail"> | null {
|
|
const linux = os === "linux";
|
|
const key = `${d.check}/${d.status}${d.reason ? `/${d.reason}` : ""}`;
|
|
switch (key) {
|
|
case "prepare/unavailable/step_missing":
|
|
return {
|
|
check: d.check,
|
|
level: "warning",
|
|
title: "The agent's step outside its sandbox didn't run",
|
|
fix: "Disk health and ZFS pools are only read in that step. Install the agent again (Servers → Manage servers has the command) so its systemd unit has it; if it is installed, look at `journalctl -u homelab-manager-agent` on the host.",
|
|
};
|
|
case "updates/unavailable/no_package_manager":
|
|
return { check: d.check, level: "info", title: "Pending updates can't be read here", fix: "The agent understands apt, dnf, zypper and pacman, and Windows Update. This machine has none of them." };
|
|
case "failed_units/unavailable/no_systemd":
|
|
return { check: d.check, level: "info", title: "Failed services can't be read: this machine doesn't run systemd", fix: null };
|
|
case "smart/unavailable/tool_missing":
|
|
return { check: d.check, level: "info", title: "Disk health isn't reported: smartctl isn't installed", fix: "Install smartmontools on the host (apt install smartmontools, or dnf install smartmontools); the next report includes it." };
|
|
case "smart/unavailable/not_root":
|
|
return { check: d.check, level: "warning", title: "Disk health needs the agent to run as root", fix: "The install script's systemd unit runs it as root. A copy started by hand as another user can't read the disks." };
|
|
case "zfs/unavailable/tool_missing":
|
|
return null; // most machines have no ZFS, and that is not a problem
|
|
case "hardware/unavailable/not_found":
|
|
return { check: d.check, level: "info", title: "No maker, model or serial number found", fix: "Many virtual machines and single-board computers don't have them. Type them into the Inventory instead." };
|
|
}
|
|
if (d.status === "failed") {
|
|
const what: Record<string, string> = { updates: "pending updates", failed_units: "services", smart: "disk health", zfs: "storage pools", host_health: "the machine's health", hardware: "what the machine is" };
|
|
const fixes: Record<string, string> = linux
|
|
? {
|
|
updates: "Run `sudo /usr/local/bin/homelab-manager-agent.sh --check-updates` on the host to see what the package manager said (dnf and zypper answer from their caches: `sudo dnf makecache`).",
|
|
failed_units: "`systemctl list-units --state=failed` didn't answer in time on that host.",
|
|
zfs: "`zpool list` didn't answer; check that the ZFS kernel module is loaded.",
|
|
}
|
|
: {
|
|
updates: "Windows Update didn't answer (the search is tried again after an hour). Check the Windows Update service and that the machine can reach Microsoft.",
|
|
failed_units: "Reading services through WMI failed. Check that the Windows Management Instrumentation service is running.",
|
|
smart: "Get-PhysicalDisk failed. In an elevated PowerShell, run `Get-PhysicalDisk` to see why.",
|
|
zfs: "Get-StoragePool failed. In an elevated PowerShell, run `Get-StoragePool` to see why.",
|
|
host_health: "Reading the machine's health failed; the detail says why.",
|
|
hardware: "Reading the system's own details through WMI failed; the detail says why.",
|
|
};
|
|
return { check: d.check, level: "warning", title: `Reading ${what[d.check] ?? d.check} failed`, fix: fixes[d.check] ?? null };
|
|
}
|
|
return { check: d.check, level: "info", title: `${d.check.replace(/_/g, " ")} isn't available${d.reason ? ` (${d.reason.replace(/_/g, " ")})` : ""}`, fix: null };
|
|
}
|
|
|
|
/** What to tell a person about one server's agent: what it couldn't collect (with what to do) and, when it is behind, that. Warnings first. */
|
|
export function describeDiagnostics(stored: StoredDiagnostics | null, context: { osType: string; lastSeenAt: string | null; agentVersion: number | null; latest: LatestAgentVersions }): DiagnosticNote[] {
|
|
const os: Os = context.osType === "windows" ? "windows" : "linux";
|
|
const notes: DiagnosticNote[] = [];
|
|
for (const d of stored?.items ?? []) {
|
|
const note = describe(d, os);
|
|
if (note) notes.push({ ...note, detail: d.status === "failed" ? (d.detail ?? null) : null }); // what the agent said is only worth showing for a failure
|
|
}
|
|
if (isAgentOutdated({ osType: context.osType, lastSeenAt: context.lastSeenAt, agentVersion: context.agentVersion }, context.latest)) {
|
|
const current = os === "windows" ? context.latest.windows : context.latest.linux;
|
|
notes.push({
|
|
check: "agent_behind",
|
|
level: "info",
|
|
title: `This agent is version ${context.agentVersion ?? 1}; ${current} is current`,
|
|
fix: "A newer agent reports more and says more about what it couldn't read. Update it from the Updates page, or switch on automatic updates for agents in Settings.",
|
|
detail: null,
|
|
});
|
|
}
|
|
return notes.sort((a, b) => Number(b.level === "warning") - Number(a.level === "warning"));
|
|
}
|