Agent diagnostics ("what the agent couldn't collect") and services to ignore per server
- Agents (Linux 6, Windows 5) note each check they couldn't make and why; the server page turns that into tips and fixes, including when the agent is behind - Linux --prepare leaves its notes for the report; a missing prepare step is itself noted - Services to ignore are set per server from the Host health card (names or * patterns), applied on the server so alerts and /metrics drop them at once; saving alert limits keeps the list Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b5384c1e03
commit
c3bc6bbf72
26 files changed
+4118
-45
No files matched your search
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* What an agent could not collect, and what to do about it. Agents (Linux version 6, Windows version 5) note each check they couldn't make —
|
||||
* with whether nothing was there to read it with ("unavailable") or it tried and couldn't ("failed"), why in a word and at most a line of
|
||||
* what the tool said. They say only problems; a check not mentioned went fine. The words and the fixes live here, not in the agents, so they
|
||||
* can be improved without installing anything again.
|
||||
*/
|
||||
import { isAgentOutdated, type LatestAgentVersions } from "./agentVersions.js";
|
||||
|
||||
export interface AgentDiagnostic {
|
||||
check: string;
|
||||
status: "unavailable" | "failed";
|
||||
reason?: string;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
export interface StoredDiagnostics {
|
||||
items: AgentDiagnostic[];
|
||||
/** ISO — the report these came from. */
|
||||
reportedAt: string;
|
||||
}
|
||||
|
||||
export interface DiagnosticNote {
|
||||
check: string;
|
||||
/** "warning": something to look at. "info": a tip, or something that is simply how this machine is. */
|
||||
level: "warning" | "info";
|
||||
title: string;
|
||||
fix: string | null;
|
||||
/** What the agent said about it, when it said anything. */
|
||||
detail: string | null;
|
||||
}
|
||||
|
||||
const clean = (v: unknown, max: number) => (typeof v === "string" ? v.replace(/[\u0000-\u001f]/g, " ").trim().slice(0, max) : "");
|
||||
|
||||
/** Reads what is stored, leniently. Null when the agent has never said (an agent that predates this, or one that has nothing to report on). */
|
||||
export function parseDiagnostics(stored: string | null | undefined): StoredDiagnostics | null {
|
||||
if (!stored) return null;
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
if (!v || typeof v !== "object" || !Array.isArray(v.items)) return null;
|
||||
const items: AgentDiagnostic[] = [];
|
||||
for (const i of v.items.slice(0, 20)) {
|
||||
if (!i || typeof i !== "object") continue;
|
||||
const check = clean(i.check, 40);
|
||||
if (!check || (i.status !== "unavailable" && i.status !== "failed")) continue;
|
||||
const item: AgentDiagnostic = { check, status: i.status };
|
||||
const reason = clean(i.reason, 40);
|
||||
const detail = clean(i.detail, 160);
|
||||
if (reason) item.reason = reason;
|
||||
if (detail) item.detail = detail;
|
||||
items.push(item);
|
||||
}
|
||||
return { items, reportedAt: typeof v.reportedAt === "string" ? v.reportedAt : "" };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** What to store after a report that carried notes (an empty list is an answer: nothing was missed). */
|
||||
export function nextDiagnostics(incoming: AgentDiagnostic[], now: string): string {
|
||||
return JSON.stringify({ items: incoming, reportedAt: now } satisfies StoredDiagnostics);
|
||||
}
|
||||
|
||||
type Os = "linux" | "windows";
|
||||
|
||||
/** One note for what the agent said, or null when it is not worth a person's attention (no ZFS on a machine that has none). */
|
||||
function describe(d: AgentDiagnostic, os: Os): Omit<DiagnosticNote, "detail"> | null {
|
||||
const linux = os === "linux";
|
||||
const key = `${d.check}/${d.status}${d.reason ? `/${d.reason}` : ""}`;
|
||||
switch (key) {
|
||||
case "prepare/unavailable/step_missing":
|
||||
return {
|
||||
check: d.check,
|
||||
level: "warning",
|
||||
title: "The agent's step outside its sandbox didn't run",
|
||||
fix: "Disk health and ZFS pools are only read in that step. Install the agent again (Servers → Manage servers has the command) so its systemd unit has it; if it is installed, look at `journalctl -u homelab-manager-agent` on the host.",
|
||||
};
|
||||
case "updates/unavailable/no_package_manager":
|
||||
return { check: d.check, level: "info", title: "Pending updates can't be read here", fix: "The agent understands apt, dnf, zypper and pacman, and Windows Update. This machine has none of them." };
|
||||
case "failed_units/unavailable/no_systemd":
|
||||
return { check: d.check, level: "info", title: "Failed services can't be read: this machine doesn't run systemd", fix: null };
|
||||
case "smart/unavailable/tool_missing":
|
||||
return { check: d.check, level: "info", title: "Disk health isn't reported: smartctl isn't installed", fix: "Install smartmontools on the host (apt install smartmontools, or dnf install smartmontools); the next report includes it." };
|
||||
case "smart/unavailable/not_root":
|
||||
return { check: d.check, level: "warning", title: "Disk health needs the agent to run as root", fix: "The install script's systemd unit runs it as root. A copy started by hand as another user can't read the disks." };
|
||||
case "zfs/unavailable/tool_missing":
|
||||
return null; // most machines have no ZFS, and that is not a problem
|
||||
case "hardware/unavailable/not_found":
|
||||
return { check: d.check, level: "info", title: "No maker, model or serial number found", fix: "Many virtual machines and single-board computers don't have them. Type them into the Inventory instead." };
|
||||
}
|
||||
if (d.status === "failed") {
|
||||
const what: Record<string, string> = { updates: "pending updates", failed_units: "services", smart: "disk health", zfs: "storage pools", host_health: "the machine's health", hardware: "what the machine is" };
|
||||
const fixes: Record<string, string> = linux
|
||||
? {
|
||||
updates: "Run `sudo /usr/local/bin/homelab-manager-agent.sh --check-updates` on the host to see what the package manager said (dnf and zypper answer from their caches: `sudo dnf makecache`).",
|
||||
failed_units: "`systemctl list-units --state=failed` didn't answer in time on that host.",
|
||||
zfs: "`zpool list` didn't answer; check that the ZFS kernel module is loaded.",
|
||||
}
|
||||
: {
|
||||
updates: "Windows Update didn't answer (the search is tried again after an hour). Check the Windows Update service and that the machine can reach Microsoft.",
|
||||
failed_units: "Reading services through WMI failed. Check that the Windows Management Instrumentation service is running.",
|
||||
smart: "Get-PhysicalDisk failed. In an elevated PowerShell, run `Get-PhysicalDisk` to see why.",
|
||||
zfs: "Get-StoragePool failed. In an elevated PowerShell, run `Get-StoragePool` to see why.",
|
||||
host_health: "Reading the machine's health failed; the detail says why.",
|
||||
hardware: "Reading the system's own details through WMI failed; the detail says why.",
|
||||
};
|
||||
return { check: d.check, level: "warning", title: `Reading ${what[d.check] ?? d.check} failed`, fix: fixes[d.check] ?? null };
|
||||
}
|
||||
return { check: d.check, level: "info", title: `${d.check.replace(/_/g, " ")} isn't available${d.reason ? ` (${d.reason.replace(/_/g, " ")})` : ""}`, fix: null };
|
||||
}
|
||||
|
||||
/** What to tell a person about one server's agent: what it couldn't collect (with what to do) and, when it is behind, that. Warnings first. */
|
||||
export function describeDiagnostics(stored: StoredDiagnostics | null, context: { osType: string; lastSeenAt: string | null; agentVersion: number | null; latest: LatestAgentVersions }): DiagnosticNote[] {
|
||||
const os: Os = context.osType === "windows" ? "windows" : "linux";
|
||||
const notes: DiagnosticNote[] = [];
|
||||
for (const d of stored?.items ?? []) {
|
||||
const note = describe(d, os);
|
||||
if (note) notes.push({ ...note, detail: d.status === "failed" ? (d.detail ?? null) : null }); // what the agent said is only worth showing for a failure
|
||||
}
|
||||
if (isAgentOutdated({ osType: context.osType, lastSeenAt: context.lastSeenAt, agentVersion: context.agentVersion }, context.latest)) {
|
||||
const current = os === "windows" ? context.latest.windows : context.latest.linux;
|
||||
notes.push({
|
||||
check: "agent_behind",
|
||||
level: "info",
|
||||
title: `This agent is version ${context.agentVersion ?? 1}; ${current} is current`,
|
||||
fix: "A newer agent reports more and says more about what it couldn't read. Update it from the Updates page, or switch on automatic updates for agents in Settings.",
|
||||
detail: null,
|
||||
});
|
||||
}
|
||||
return notes.sort((a, b) => Number(b.level === "warning") - Number(a.level === "warning"));
|
||||
}
|
||||
Reference in new issue
Block a user