Agent diagnostics ("what the agent couldn't collect") and services to ignore per server
- Agents (Linux 6, Windows 5) note each check they couldn't make and why; the server page turns that into tips and fixes, including when the agent is behind - Linux --prepare leaves its notes for the report; a missing prepare step is itself noted - Services to ignore are set per server from the Host health card (names or * patterns), applied on the server so alerts and /metrics drop them at once; saving alert limits keeps the list Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b5384c1e03
commit
c3bc6bbf72
26 files changed
+4118
-45
No files matched your search
@@ -0,0 +1 @@
|
||||
ALTER TABLE `servers` ADD `agent_diagnostics` text;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -267,6 +267,13 @@
|
||||
"when": 1792344782975,
|
||||
"tag": "0037_past_magik",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 38,
|
||||
"version": "6",
|
||||
"when": 1792354148200,
|
||||
"tag": "0038_wise_malcolm_colcord",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -341,6 +341,9 @@ export const servers = sqliteTable("servers", {
|
||||
// What the agent says about the machine's own health, as JSON: {failedUnits, smart, zfs, reportedAt} — see services/hostHealth.ts. Null until it says.
|
||||
hostHealth: text("host_health"),
|
||||
|
||||
// What the agent says it could not collect, as JSON: {items: [{check, status, reason?, detail?}], reportedAt} — see services/agentDiagnostics.ts. Null until an agent that can say so has.
|
||||
agentDiagnostics: text("agent_diagnostics"),
|
||||
|
||||
// What the machine is, as its firmware says (agents report it): maker, model and serial number. Used to fill in the inventory.
|
||||
hwManufacturer: text("hw_manufacturer"),
|
||||
hwModel: text("hw_model"),
|
||||
|
||||
@@ -59,7 +59,23 @@ const hardwareSchema = z.object({
|
||||
serial: text(100).optional().catch(undefined),
|
||||
});
|
||||
|
||||
// What the agent could not collect: leniently read, each entry on its own, so one odd one never costs the report.
|
||||
const diagnosticsSchema = z.unknown().transform((v) => {
|
||||
if (!Array.isArray(v)) return undefined;
|
||||
const out: { check: string; status: "unavailable" | "failed"; reason?: string; detail?: string }[] = [];
|
||||
for (const i of v.slice(0, 20)) {
|
||||
if (!i || typeof i !== "object") continue;
|
||||
const e = i as Record<string, unknown>;
|
||||
const clean = (x: unknown, max: number) => (typeof x === "string" ? x.replace(/[\u0000-\u001f]/g, " ").trim().slice(0, max) : "");
|
||||
const check = clean(e.check, 40);
|
||||
if (!check || (e.status !== "unavailable" && e.status !== "failed")) continue;
|
||||
out.push({ check, status: e.status, ...(clean(e.reason, 40) ? { reason: clean(e.reason, 40) } : {}), ...(clean(e.detail, 160) ? { detail: clean(e.detail, 160) } : {}) });
|
||||
}
|
||||
return out;
|
||||
});
|
||||
|
||||
const systemSchema = z.object({
|
||||
diagnostics: diagnosticsSchema.optional(),
|
||||
hardware: z
|
||||
.unknown()
|
||||
.optional()
|
||||
|
||||
@@ -8,7 +8,8 @@ import { generateApiToken } from "../services/tokens.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js";
|
||||
import { parsePendingUpdates } from "../services/pendingUpdates.js";
|
||||
import { parseHostHealth } from "../services/hostHealth.js";
|
||||
import { markIgnoredServices, parseHostHealth } from "../services/hostHealth.js";
|
||||
import { describeDiagnostics, parseDiagnostics } from "../services/agentDiagnostics.js";
|
||||
import { checkThresholds, parseThresholds } from "../services/serverThresholds.js";
|
||||
import { latestAgentVersions } from "../services/agentVersions.js";
|
||||
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
|
||||
@@ -27,8 +28,8 @@ import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
export const serversRouter = Router();
|
||||
|
||||
/** A server row as the API returns it: no token hash, and tags as an array rather than the stored JSON. */
|
||||
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null; hostHealth: string | null; thresholds: string | null }>(row: T) {
|
||||
const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, thresholds, ...rest } = row;
|
||||
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null; hostHealth: string | null; thresholds: string | null; agentDiagnostics?: string | null }>(row: T) {
|
||||
const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, thresholds, agentDiagnostics: _diagnostics, ...rest } = row;
|
||||
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth), thresholds: parseThresholds(thresholds) };
|
||||
}
|
||||
|
||||
@@ -382,6 +383,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
|
||||
pendingUpdates: rawUpdates,
|
||||
hostHealth: rawHealth,
|
||||
thresholds: rawThresholds,
|
||||
agentDiagnostics: rawDiagnostics,
|
||||
...serverOut
|
||||
} = server;
|
||||
|
||||
@@ -395,7 +397,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
|
||||
children: [...links2.entries()].filter(([, l]) => l.parentId === id).map(([childId, l]) => ({ id: childId, name: nameOf.get(childId) ?? "", auto: l.auto })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
};
|
||||
|
||||
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: parseHostHealth(rawHealth), thresholds: parseThresholds(rawThresholds) }, thresholdDefaults: (await getSettings()).healthChecks, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
|
||||
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: markIgnoredServices(parseHostHealth(rawHealth), parseThresholds(rawThresholds)?.ignoreServices), thresholds: parseThresholds(rawThresholds) }, diagnostics: describeDiagnostics(parseDiagnostics(rawDiagnostics), { osType: server.osType, lastSeenAt: server.lastSeenAt, agentVersion: server.agentVersion, latest: latestAgentVersions() }), thresholdDefaults: (await getSettings()).healthChecks, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
|
||||
}));
|
||||
|
||||
const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) });
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* What an agent could not collect, and what to do about it. Agents (Linux version 6, Windows version 5) note each check they couldn't make —
|
||||
* with whether nothing was there to read it with ("unavailable") or it tried and couldn't ("failed"), why in a word and at most a line of
|
||||
* what the tool said. They say only problems; a check not mentioned went fine. The words and the fixes live here, not in the agents, so they
|
||||
* can be improved without installing anything again.
|
||||
*/
|
||||
import { isAgentOutdated, type LatestAgentVersions } from "./agentVersions.js";
|
||||
|
||||
export interface AgentDiagnostic {
|
||||
check: string;
|
||||
status: "unavailable" | "failed";
|
||||
reason?: string;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
export interface StoredDiagnostics {
|
||||
items: AgentDiagnostic[];
|
||||
/** ISO — the report these came from. */
|
||||
reportedAt: string;
|
||||
}
|
||||
|
||||
export interface DiagnosticNote {
|
||||
check: string;
|
||||
/** "warning": something to look at. "info": a tip, or something that is simply how this machine is. */
|
||||
level: "warning" | "info";
|
||||
title: string;
|
||||
fix: string | null;
|
||||
/** What the agent said about it, when it said anything. */
|
||||
detail: string | null;
|
||||
}
|
||||
|
||||
const clean = (v: unknown, max: number) => (typeof v === "string" ? v.replace(/[\u0000-\u001f]/g, " ").trim().slice(0, max) : "");
|
||||
|
||||
/** Reads what is stored, leniently. Null when the agent has never said (an agent that predates this, or one that has nothing to report on). */
|
||||
export function parseDiagnostics(stored: string | null | undefined): StoredDiagnostics | null {
|
||||
if (!stored) return null;
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
if (!v || typeof v !== "object" || !Array.isArray(v.items)) return null;
|
||||
const items: AgentDiagnostic[] = [];
|
||||
for (const i of v.items.slice(0, 20)) {
|
||||
if (!i || typeof i !== "object") continue;
|
||||
const check = clean(i.check, 40);
|
||||
if (!check || (i.status !== "unavailable" && i.status !== "failed")) continue;
|
||||
const item: AgentDiagnostic = { check, status: i.status };
|
||||
const reason = clean(i.reason, 40);
|
||||
const detail = clean(i.detail, 160);
|
||||
if (reason) item.reason = reason;
|
||||
if (detail) item.detail = detail;
|
||||
items.push(item);
|
||||
}
|
||||
return { items, reportedAt: typeof v.reportedAt === "string" ? v.reportedAt : "" };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** What to store after a report that carried notes (an empty list is an answer: nothing was missed). */
|
||||
export function nextDiagnostics(incoming: AgentDiagnostic[], now: string): string {
|
||||
return JSON.stringify({ items: incoming, reportedAt: now } satisfies StoredDiagnostics);
|
||||
}
|
||||
|
||||
type Os = "linux" | "windows";
|
||||
|
||||
/** One note for what the agent said, or null when it is not worth a person's attention (no ZFS on a machine that has none). */
|
||||
function describe(d: AgentDiagnostic, os: Os): Omit<DiagnosticNote, "detail"> | null {
|
||||
const linux = os === "linux";
|
||||
const key = `${d.check}/${d.status}${d.reason ? `/${d.reason}` : ""}`;
|
||||
switch (key) {
|
||||
case "prepare/unavailable/step_missing":
|
||||
return {
|
||||
check: d.check,
|
||||
level: "warning",
|
||||
title: "The agent's step outside its sandbox didn't run",
|
||||
fix: "Disk health and ZFS pools are only read in that step. Install the agent again (Servers → Manage servers has the command) so its systemd unit has it; if it is installed, look at `journalctl -u homelab-manager-agent` on the host.",
|
||||
};
|
||||
case "updates/unavailable/no_package_manager":
|
||||
return { check: d.check, level: "info", title: "Pending updates can't be read here", fix: "The agent understands apt, dnf, zypper and pacman, and Windows Update. This machine has none of them." };
|
||||
case "failed_units/unavailable/no_systemd":
|
||||
return { check: d.check, level: "info", title: "Failed services can't be read: this machine doesn't run systemd", fix: null };
|
||||
case "smart/unavailable/tool_missing":
|
||||
return { check: d.check, level: "info", title: "Disk health isn't reported: smartctl isn't installed", fix: "Install smartmontools on the host (apt install smartmontools, or dnf install smartmontools); the next report includes it." };
|
||||
case "smart/unavailable/not_root":
|
||||
return { check: d.check, level: "warning", title: "Disk health needs the agent to run as root", fix: "The install script's systemd unit runs it as root. A copy started by hand as another user can't read the disks." };
|
||||
case "zfs/unavailable/tool_missing":
|
||||
return null; // most machines have no ZFS, and that is not a problem
|
||||
case "hardware/unavailable/not_found":
|
||||
return { check: d.check, level: "info", title: "No maker, model or serial number found", fix: "Many virtual machines and single-board computers don't have them. Type them into the Inventory instead." };
|
||||
}
|
||||
if (d.status === "failed") {
|
||||
const what: Record<string, string> = { updates: "pending updates", failed_units: "services", smart: "disk health", zfs: "storage pools", host_health: "the machine's health", hardware: "what the machine is" };
|
||||
const fixes: Record<string, string> = linux
|
||||
? {
|
||||
updates: "Run `sudo /usr/local/bin/homelab-manager-agent.sh --check-updates` on the host to see what the package manager said (dnf and zypper answer from their caches: `sudo dnf makecache`).",
|
||||
failed_units: "`systemctl list-units --state=failed` didn't answer in time on that host.",
|
||||
zfs: "`zpool list` didn't answer; check that the ZFS kernel module is loaded.",
|
||||
}
|
||||
: {
|
||||
updates: "Windows Update didn't answer (the search is tried again after an hour). Check the Windows Update service and that the machine can reach Microsoft.",
|
||||
failed_units: "Reading services through WMI failed. Check that the Windows Management Instrumentation service is running.",
|
||||
smart: "Get-PhysicalDisk failed. In an elevated PowerShell, run `Get-PhysicalDisk` to see why.",
|
||||
zfs: "Get-StoragePool failed. In an elevated PowerShell, run `Get-StoragePool` to see why.",
|
||||
host_health: "Reading the machine's health failed; the detail says why.",
|
||||
hardware: "Reading the system's own details through WMI failed; the detail says why.",
|
||||
};
|
||||
return { check: d.check, level: "warning", title: `Reading ${what[d.check] ?? d.check} failed`, fix: fixes[d.check] ?? null };
|
||||
}
|
||||
return { check: d.check, level: "info", title: `${d.check.replace(/_/g, " ")} isn't available${d.reason ? ` (${d.reason.replace(/_/g, " ")})` : ""}`, fix: null };
|
||||
}
|
||||
|
||||
/** What to tell a person about one server's agent: what it couldn't collect (with what to do) and, when it is behind, that. Warnings first. */
|
||||
export function describeDiagnostics(stored: StoredDiagnostics | null, context: { osType: string; lastSeenAt: string | null; agentVersion: number | null; latest: LatestAgentVersions }): DiagnosticNote[] {
|
||||
const os: Os = context.osType === "windows" ? "windows" : "linux";
|
||||
const notes: DiagnosticNote[] = [];
|
||||
for (const d of stored?.items ?? []) {
|
||||
const note = describe(d, os);
|
||||
if (note) notes.push({ ...note, detail: d.status === "failed" ? (d.detail ?? null) : null }); // what the agent said is only worth showing for a failure
|
||||
}
|
||||
if (isAgentOutdated({ osType: context.osType, lastSeenAt: context.lastSeenAt, agentVersion: context.agentVersion }, context.latest)) {
|
||||
const current = os === "windows" ? context.latest.windows : context.latest.linux;
|
||||
notes.push({
|
||||
check: "agent_behind",
|
||||
level: "info",
|
||||
title: `This agent is version ${context.agentVersion ?? 1}; ${current} is current`,
|
||||
fix: "A newer agent reports more and says more about what it couldn't read. Update it from the Updates page, or switch on automatic updates for agents in Settings.",
|
||||
detail: null,
|
||||
});
|
||||
}
|
||||
return notes.sort((a, b) => Number(b.level === "warning") - Number(a.level === "warning"));
|
||||
}
|
||||
@@ -11,6 +11,7 @@
|
||||
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too.
|
||||
* Windows: also reports failed services, disk health and Storage Spaces pools, and what the machine is (maker, model, serial)
|
||||
* 5 — Linux: also says what the machine is (maker, model, serial), for the inventory
|
||||
* 6 — Linux (and Windows 5): also says what it could not collect and why, so the server can say what to fix
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
@@ -10,7 +10,7 @@ import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.j
|
||||
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
|
||||
import { dependencyCover, HEALTH_STATE_FLAG } from "./dependencies.js";
|
||||
import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { hostHealthConditions, parseHostHealth, type HostHealth } from "./hostHealth.js";
|
||||
import { hostHealthConditions, parseHostHealth, withoutIgnoredServices, type HostHealth } from "./hostHealth.js";
|
||||
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
|
||||
|
||||
const STATE_FLAG = HEALTH_STATE_FLAG;
|
||||
@@ -306,7 +306,7 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
} catch {
|
||||
// a malformed report shouldn't stop every other server being checked
|
||||
}
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: parseHostHealth(s.hostHealth), thresholds: parseThresholds(s.thresholds) });
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: withoutIgnoredServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.ignoreServices), thresholds: parseThresholds(s.thresholds) });
|
||||
}
|
||||
|
||||
const rows = await db
|
||||
|
||||
@@ -8,6 +8,8 @@
|
||||
export interface FailedUnit {
|
||||
unit: string;
|
||||
description: string;
|
||||
/** Set where it is shown, for a unit this server has been told to ignore. It never counts as failed. */
|
||||
ignored?: boolean;
|
||||
}
|
||||
export interface SmartDisk {
|
||||
device: string;
|
||||
@@ -82,6 +84,31 @@ export function nextHostHealth(previous: string | null, incoming: IncomingHostHe
|
||||
return any ? JSON.stringify(next) : previous;
|
||||
}
|
||||
|
||||
const globToRegex = (pattern: string) => new RegExp(`^${pattern.trim().toLowerCase().replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*")}$`);
|
||||
|
||||
/** Whether a unit is one this server is told to ignore: a name, or a pattern with * ("gupdate*"); case doesn't matter and ".service" may be left off. */
|
||||
export function isServiceIgnored(unit: string, patterns: string[] | undefined): boolean {
|
||||
if (!patterns || patterns.length === 0) return false;
|
||||
const u = unit.toLowerCase();
|
||||
const bare = u.replace(/\.service$/, "");
|
||||
return patterns.some((p) => {
|
||||
const re = globToRegex(p);
|
||||
return re.test(u) || re.test(bare);
|
||||
});
|
||||
}
|
||||
|
||||
/** The health with the ignored services taken out: what alerts and figures are worked from. */
|
||||
export function withoutIgnoredServices(health: HostHealth | null, patterns: string[] | undefined): HostHealth | null {
|
||||
if (!health?.failedUnits || !patterns || patterns.length === 0) return health;
|
||||
return { ...health, failedUnits: health.failedUnits.filter((u) => !isServiceIgnored(u.unit, patterns)) };
|
||||
}
|
||||
|
||||
/** The health with the ignored services marked, not removed: what the server's page shows, so they can be seen and taken off the list. */
|
||||
export function markIgnoredServices(health: HostHealth | null, patterns: string[] | undefined): HostHealth | null {
|
||||
if (!health?.failedUnits || !patterns || patterns.length === 0) return health;
|
||||
return { ...health, failedUnits: health.failedUnits.map((u) => (isServiceIgnored(u.unit, patterns) ? { ...u, ignored: true } : u)) };
|
||||
}
|
||||
|
||||
/** A pool in any state but ONLINE has lost something (DEGRADED has lost redundancy, the rest are worse). */
|
||||
export const poolIsFine = (p: ZfsPool) => p.health.toUpperCase() === "ONLINE";
|
||||
|
||||
|
||||
@@ -10,7 +10,8 @@ import { getAlerts } from "./alerts.js";
|
||||
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
|
||||
import { getSourceStatus } from "./integrationHealthMonitor.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
|
||||
import { parseHostHealth, poolIsFine, withoutIgnoredServices } from "./hostHealth.js";
|
||||
import { parseThresholds as parseServerThresholds } from "./serverThresholds.js";
|
||||
import { parseThresholds } from "./serverThresholds.js";
|
||||
import { getCachedBackupCoverage } from "./backupCoverage.js";
|
||||
import { lastAttempt as lastSelfBackup, listSnapshots } from "./selfBackup.js";
|
||||
@@ -88,7 +89,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
if (updates.security !== null) out.add("homelab_server_pending_security_updates", "gauge", "Of those, the ones that are security fixes.", updates.security, { server });
|
||||
out.add("homelab_server_reboot_required", "gauge", "1 if a restart is due.", updates.rebootRequired ? 1 : 0, { server });
|
||||
}
|
||||
const health = parseHostHealth(s.hostHealth);
|
||||
const health = withoutIgnoredServices(parseHostHealth(s.hostHealth), parseServerThresholds(s.thresholds)?.ignoreServices);
|
||||
if (health?.failedUnits) out.add("homelab_server_failed_units", "gauge", "Services (systemd units) that have failed on the server.", health.failedUnits.length, { server });
|
||||
for (const d of health?.smart ?? []) {
|
||||
if (d.healthy !== null) out.add("homelab_server_disk_healthy", "gauge", "1 if the disk's own SMART self-assessment passes, 0 if it fails.", d.healthy ? 1 : 0, { server, device: d.device });
|
||||
|
||||
@@ -21,6 +21,8 @@ export interface ServerThresholds {
|
||||
/** 0 turns the stale-backup alert off for this server's guest. */
|
||||
backupStaleHours?: number;
|
||||
mounts?: Record<string, MountThreshold>;
|
||||
/** Services (units) never to count as failed on this server: names or patterns like "gupdate*" (case does not matter, ".service" is optional). */
|
||||
ignoreServices?: string[];
|
||||
}
|
||||
|
||||
/** The global values these override. */
|
||||
@@ -38,6 +40,8 @@ export const LIMITS = {
|
||||
backupStaleHours: { min: 0, max: 2160 },
|
||||
} as const;
|
||||
export const MAX_MOUNT_RULES = 50;
|
||||
export const MAX_IGNORED_SERVICES = 30;
|
||||
const MAX_SERVICE_PATTERN = 100;
|
||||
|
||||
const inRange = (v: unknown, key: keyof typeof LIMITS): v is number => typeof v === "number" && Number.isInteger(v) && v >= LIMITS[key].min && v <= LIMITS[key].max;
|
||||
|
||||
@@ -61,6 +65,11 @@ export function parseThresholds(stored: string | null | undefined): ServerThresh
|
||||
}
|
||||
if (Object.keys(mounts).length > 0) out.mounts = mounts;
|
||||
}
|
||||
if (Array.isArray(v.ignoreServices)) {
|
||||
const strings: string[] = (v.ignoreServices as unknown[]).filter((p): p is string => typeof p === "string");
|
||||
const list = [...new Set(strings.map((p) => p.trim()).filter((p) => p.length > 0 && p.length <= MAX_SERVICE_PATTERN))].slice(0, MAX_IGNORED_SERVICES);
|
||||
if (list.length > 0) out.ignoreServices = list;
|
||||
}
|
||||
return Object.keys(out).length > 0 ? out : null;
|
||||
} catch {
|
||||
return null;
|
||||
@@ -103,6 +112,20 @@ export function checkThresholds(input: unknown): { ok: true; value: ServerThresh
|
||||
}
|
||||
if (Object.keys(mounts).length > 0) out.mounts = mounts;
|
||||
}
|
||||
if (v.ignoreServices !== undefined && v.ignoreServices !== null) {
|
||||
if (!Array.isArray(v.ignoreServices)) return { ok: false, message: "The services to ignore should be a list." };
|
||||
const list: string[] = [];
|
||||
for (const raw of v.ignoreServices) {
|
||||
if (typeof raw !== "string") return { ok: false, message: "Each service to ignore should be a name or a pattern." };
|
||||
const p = raw.trim();
|
||||
if (p.length === 0) continue;
|
||||
// eslint-disable-next-line no-control-regex
|
||||
if (p.length > MAX_SERVICE_PATTERN || /[\u0000-\u001f]/.test(p)) return { ok: false, message: `"${p.slice(0, 30)}…" isn't a service name or pattern (up to ${MAX_SERVICE_PATTERN} characters, no line breaks).` };
|
||||
if (!list.includes(p)) list.push(p);
|
||||
}
|
||||
if (list.length > MAX_IGNORED_SERVICES) return { ok: false, message: `At most ${MAX_IGNORED_SERVICES} services can be ignored on one server.` };
|
||||
if (list.length > 0) out.ignoreServices = list;
|
||||
}
|
||||
return { ok: true, value: Object.keys(out).length > 0 ? out : null };
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
|
||||
import { nextHostHealth, type IncomingHostHealth } from "./hostHealth.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { fillInventoryFromServer } from "./inventoryAutofill.js";
|
||||
import { nextDiagnostics, type AgentDiagnostic } from "./agentDiagnostics.js";
|
||||
|
||||
export interface IncomingTask {
|
||||
scheduleType: "cron" | "systemd_timer" | "windows_task";
|
||||
@@ -30,6 +31,8 @@ export interface IncomingSystemInfo {
|
||||
health?: IncomingHostHealth;
|
||||
/** What the machine is, from its firmware: each part present only if it says anything. */
|
||||
hardware?: { manufacturer?: string; model?: string; serial?: string };
|
||||
/** What the agent could not collect; an empty list says nothing was missed. */
|
||||
diagnostics?: AgentDiagnostic[];
|
||||
}
|
||||
|
||||
export interface AgentReport {
|
||||
@@ -118,6 +121,7 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
// Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before.
|
||||
...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}),
|
||||
...(system?.health && before ? { hostHealth: nextHostHealth(before.hostHealth, system.health, now) } : {}),
|
||||
...(system?.diagnostics ? { agentDiagnostics: nextDiagnostics(system.diagnostics, now) } : {}),
|
||||
...(system?.hardware?.manufacturer ? { hwManufacturer: system.hardware.manufacturer } : {}),
|
||||
...(system?.hardware?.model ? { hwModel: system.hardware.model } : {}),
|
||||
...(system?.hardware?.serial ? { hwSerial: system.hardware.serial } : {}),
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* What an agent could not collect and what to do about it, and the services a server is told to ignore.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-diag-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { describeDiagnostics, parseDiagnostics } = await import("../src/services/agentDiagnostics.js");
|
||||
const { isServiceIgnored, withoutIgnoredServices, markIgnoredServices } = await import("../src/services/hostHealth.js");
|
||||
const { checkThresholds, parseThresholds } = await import("../src/services/serverThresholds.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { serversRouter } = await import("../src/routes/servers.js");
|
||||
const { agentReportRouter } = await import("../src/routes/agentReport.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const latest = { linux: 6, windows: 5 };
|
||||
const ctx = (extra: Partial<{ osType: string; lastSeenAt: string | null; agentVersion: number | null }> = {}) => ({ osType: "linux", lastSeenAt: "2026-10-18T10:00:00Z", agentVersion: 6, latest, ...extra });
|
||||
const stored = (items: unknown[]) => JSON.stringify({ items, reportedAt: "2026-10-18T10:00:00Z" });
|
||||
|
||||
describe("saying what to do about what couldn't be collected", () => {
|
||||
test("a missing tool is a tip with the fix; ZFS that isn't there is not worth saying", () => {
|
||||
const notes = describeDiagnostics(
|
||||
parseDiagnostics(stored([
|
||||
{ check: "smart", status: "unavailable", reason: "tool_missing", detail: "smartctl" },
|
||||
{ check: "zfs", status: "unavailable", reason: "tool_missing", detail: "zpool" },
|
||||
{ check: "hardware", status: "unavailable", reason: "not_found" },
|
||||
])),
|
||||
ctx(),
|
||||
);
|
||||
assert.deepEqual(notes.map((n) => n.check), ["smart", "hardware"], "no word about ZFS");
|
||||
assert.equal(notes[0].level, "info");
|
||||
assert.match(notes[0].fix!, /smartmontools/);
|
||||
});
|
||||
|
||||
test("a failure is a warning that says what the agent said, with a fix for the system it is on", () => {
|
||||
const failed = stored([{ check: "updates", status: "failed", reason: "error", detail: "apt did not answer" }]);
|
||||
const linux = describeDiagnostics(parseDiagnostics(failed), ctx())[0];
|
||||
assert.equal(linux.level, "warning");
|
||||
assert.equal(linux.detail, "apt did not answer");
|
||||
assert.match(linux.fix!, /--check-updates/);
|
||||
const windows = describeDiagnostics(parseDiagnostics(failed), ctx({ osType: "windows", agentVersion: 5 }))[0];
|
||||
assert.match(windows.fix!, /Windows Update/);
|
||||
});
|
||||
|
||||
test("a step that didn't run is a warning, and warnings come first", () => {
|
||||
const notes = describeDiagnostics(parseDiagnostics(stored([{ check: "hardware", status: "unavailable", reason: "not_found" }, { check: "prepare", status: "unavailable", reason: "step_missing" }])), ctx());
|
||||
assert.deepEqual(notes.map((n) => [n.check, n.level]), [["prepare", "warning"], ["hardware", "info"]]);
|
||||
});
|
||||
|
||||
test("something it has no words for is still said, rather than dropped", () => {
|
||||
const notes = describeDiagnostics(parseDiagnostics(stored([{ check: "gpu_temps", status: "failed", reason: "error" }, { check: "fans", status: "unavailable", reason: "no_sensor" }])), ctx());
|
||||
assert.equal(notes[0].level, "warning");
|
||||
assert.match(notes[0].title, /gpu_temps/);
|
||||
assert.match(notes[1].title, /fans isn't available \(no sensor\)/);
|
||||
});
|
||||
|
||||
test("an agent that is behind is told so, whether or not it said anything", () => {
|
||||
const notes = describeDiagnostics(null, ctx({ agentVersion: 3 }));
|
||||
assert.equal(notes.length, 1);
|
||||
assert.match(notes[0].title, /version 3; 6 is current/);
|
||||
assert.deepEqual(describeDiagnostics(null, ctx()), [], "an agent that is current and silent has nothing to say");
|
||||
assert.deepEqual(describeDiagnostics(null, ctx({ lastSeenAt: null, agentVersion: 3 })), [], "a server that has never reported has no agent to update");
|
||||
});
|
||||
|
||||
test("what is stored is read leniently", () => {
|
||||
assert.equal(parseDiagnostics(null), null);
|
||||
assert.equal(parseDiagnostics("not json"), null);
|
||||
const read = parseDiagnostics(stored([{ check: "ok", status: "fine" }, "x", { check: "smart", status: "failed", detail: "a\nb" }]))!;
|
||||
assert.deepEqual(read.items, [{ check: "smart", status: "failed", detail: "a b" }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("services a server is told to ignore", () => {
|
||||
test("names and patterns, in any case, with or without .service", () => {
|
||||
assert.equal(isServiceIgnored("gupdate.service", ["gupdate"]), true);
|
||||
assert.equal(isServiceIgnored("GUpdate", ["gupdate*"]), true);
|
||||
assert.equal(isServiceIgnored("snap.lxd.activate.service", ["snap.*"]), true);
|
||||
assert.equal(isServiceIgnored("nginx.service", ["gupdate*", "snap.*"]), false);
|
||||
assert.equal(isServiceIgnored("nginx.service", ["nginx.service"]), true);
|
||||
assert.equal(isServiceIgnored("mynginx.service", ["nginx"]), false, "a name is the whole name");
|
||||
assert.equal(isServiceIgnored("a+b.service", ["a+b"]), true, "characters that mean something to a regex are only characters here");
|
||||
assert.equal(isServiceIgnored("anything", []), false);
|
||||
assert.equal(isServiceIgnored("anything", undefined), false);
|
||||
});
|
||||
|
||||
test("left out for what is judged, marked for what is shown", () => {
|
||||
const health = { failedUnits: [{ unit: "nginx.service", description: "" }, { unit: "gupdate", description: "Google Update" }], reportedAt: "t" };
|
||||
assert.deepEqual(withoutIgnoredServices(health, ["gupdate*"])!.failedUnits!.map((u) => u.unit), ["nginx.service"]);
|
||||
assert.deepEqual(markIgnoredServices(health, ["gupdate*"])!.failedUnits!.map((u) => [u.unit, u.ignored ?? false]), [["nginx.service", false], ["gupdate", true]]);
|
||||
assert.equal(withoutIgnoredServices(null, ["x"]), null);
|
||||
assert.deepEqual(withoutIgnoredServices(health, []), health);
|
||||
});
|
||||
|
||||
test("the list is part of a server's own thresholds: kept tidy, and a wrong one is refused", () => {
|
||||
const ok = checkThresholds({ ignoreServices: [" gupdate* ", "gupdate*", "", "snap.*"] });
|
||||
assert.deepEqual(ok.ok && ok.value, { ignoreServices: ["gupdate*", "snap.*"] });
|
||||
assert.equal(checkThresholds({ ignoreServices: "gupdate" }).ok, false);
|
||||
assert.equal(checkThresholds({ ignoreServices: [3] }).ok, false);
|
||||
assert.equal(checkThresholds({ ignoreServices: ["x".repeat(101)] }).ok, false);
|
||||
assert.equal(checkThresholds({ ignoreServices: Array.from({ length: 31 }, (_, i) => `s${i}`) }).ok, false);
|
||||
assert.equal(checkThresholds({ ignoreServices: [] }).ok && (checkThresholds({ ignoreServices: [] }) as { value: unknown }).value, null, "an empty list is no thresholds at all");
|
||||
assert.deepEqual(parseThresholds(JSON.stringify({ ignoreServices: ["a", 3, "", "a"] })), { ignoreServices: ["a"] });
|
||||
});
|
||||
});
|
||||
|
||||
describe("through a report, the page, the alerts and the figures", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let id = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "admin", name: "Ada", role: "admin" }]);
|
||||
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: new Date().toISOString(), agentVersion: 6 }).returning();
|
||||
id = s.id;
|
||||
const a = express();
|
||||
a.use("/api/agent/report", agentReportRouter);
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/servers", serversRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const report = (system: unknown) =>
|
||||
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", agent_version: 6, tasks: [], system }) });
|
||||
const detail = async () => await (await fetch(`${base}/servers/${id}/detail`)).json();
|
||||
const patch = (body: unknown) => fetch(`${base}/servers/${id}`, { method: "PATCH", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
|
||||
|
||||
test("what the agent says it couldn't collect is shown with what to do; garbage entries are dropped; silence keeps what was said", async () => {
|
||||
assert.equal(
|
||||
(await report({ diagnostics: [{ check: "smart", status: "unavailable", reason: "tool_missing", detail: "smartctl" }, { check: "x", status: "bogus" }, "junk", { status: "failed" }] })).status,
|
||||
202,
|
||||
);
|
||||
let d = (await detail()).diagnostics;
|
||||
assert.deepEqual(d.map((n: { check: string }) => n.check), ["smart"]);
|
||||
assert.match(d[0].fix, /smartmontools/);
|
||||
|
||||
assert.equal((await report({ cpu: { cores: 2 } })).status, 202);
|
||||
assert.equal((await detail()).diagnostics.length, 1, "a report that doesn't say keeps what was said");
|
||||
assert.equal((await report({ diagnostics: "garbage" })).status, 202);
|
||||
assert.equal((await detail()).diagnostics.length, 1, "and so does one that isn't a list");
|
||||
|
||||
assert.equal((await report({ diagnostics: [] })).status, 202);
|
||||
assert.deepEqual((await detail()).diagnostics, [], "an empty list says nothing was missed");
|
||||
const row = (await (await fetch(`${base}/servers`)).json()).servers.find((x: { id: number }) => x.id === id);
|
||||
assert.equal(row.agentDiagnostics, undefined, "never as raw stored text");
|
||||
});
|
||||
|
||||
test("a service the server is told to ignore stops being an alert and a figure, still shows on the page marked, and can be un-ignored", async () => {
|
||||
assert.equal((await report({ health: { failed_units: [{ unit: "nginx.service", description: "web" }, { unit: "gupdate.service", description: "Google Update" }] } })).status, 202);
|
||||
invalidateAlertsCache();
|
||||
const failing = async () => (await getAlerts(true)).alerts.find((a) => a.id.startsWith("services:failed-units") && a.id.includes(`:server:${id}`));
|
||||
assert.match((await failing())!.message, /2 services have failed/);
|
||||
|
||||
assert.equal((await patch({ thresholds: { ignoreServices: ["gupdate*"] } })).status, 200);
|
||||
assert.match((await failing())!.message, /1 service has failed — nginx\.service/, "the alert cache was dropped when the list changed");
|
||||
assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\\{server="elsa"[^}]*\\} 1`));
|
||||
|
||||
const units = (await detail()).server.hostHealth.failedUnits;
|
||||
assert.deepEqual(units.map((u: { unit: string; ignored?: boolean }) => [u.unit, u.ignored ?? false]), [["nginx.service", false], ["gupdate.service", true]]);
|
||||
|
||||
assert.equal((await patch({ thresholds: { ignoreServices: ["gupdate*", "nginx"] } })).status, 200);
|
||||
assert.equal(await failing(), undefined, "with every failed service ignored there is nothing to alert about");
|
||||
|
||||
assert.equal((await patch({ thresholds: null })).status, 200);
|
||||
assert.match((await failing())!.message, /2 services have failed/);
|
||||
});
|
||||
|
||||
test("a list that isn't one is refused with a message", async () => {
|
||||
const res = await patch({ thresholds: { ignoreServices: "gupdate" } });
|
||||
assert.equal(res.status, 400);
|
||||
assert.match((await res.json()).message, /should be a list/);
|
||||
});
|
||||
});
|
||||
@@ -32,9 +32,9 @@ function run(snippet: string): unknown {
|
||||
}
|
||||
|
||||
describe("the Windows agent's health parts", { skip: !havePwsh }, () => {
|
||||
test("it says it is version 4, and stays plain ASCII", () => {
|
||||
test("it says it is version 5, and stays plain ASCII", () => {
|
||||
const text = readFileSync(script, "utf8");
|
||||
assert.match(text, /^\$script:AgentVersion = 4\s*$/m);
|
||||
assert.match(text, /^\$script:AgentVersion = 5\s*$/m);
|
||||
assert.ok(!/[^\x00-\x7f]/.test(text));
|
||||
});
|
||||
|
||||
@@ -103,6 +103,22 @@ describe("the Windows agent's health parts", { skip: !havePwsh }, () => {
|
||||
assert.deepEqual(out.map((p) => [p.name, p.health, p.capacity_percent]), [["Media", "ONLINE", 25], ["Backups", "DEGRADED", 90], ["Broken", "FAULTED", null]]);
|
||||
});
|
||||
|
||||
test("what could not be read is noted, with a word for why and at most a line of what Windows said", () => {
|
||||
const out = run(`
|
||||
function Get-CimInstance { throw "WMI is unhappy\`r\`nsecond line" }
|
||||
$WarningPreference = 'SilentlyContinue'
|
||||
$null = Get-HostHealth @{ IgnoreServices = @() }
|
||||
Add-Diag 'hardware' 'unavailable' 'not_found'
|
||||
Add-Diag 'updates' 'failed' 'error' ('x' * 400)
|
||||
ConvertTo-Json -InputObject @{ items = @($script:Diag) } -Depth 4 -Compress
|
||||
`) as { items: { check: string; status: string; reason?: string; detail?: string }[] };
|
||||
const byCheck = Object.fromEntries(out.items.map((i) => [i.check, i]));
|
||||
assert.deepEqual([byCheck.failed_units.status, byCheck.failed_units.reason], ["failed", "error"]);
|
||||
assert.equal(byCheck.failed_units.detail, "WMI is unhappy second line", "line breaks are flattened");
|
||||
assert.deepEqual(byCheck.hardware, { check: "hardware", status: "unavailable", reason: "not_found" }, "nothing to say about it, no detail");
|
||||
assert.equal(byCheck.updates.detail!.length, 160, "cut to a line");
|
||||
});
|
||||
|
||||
test("a maker's placeholders aren't a model or a serial number", () => {
|
||||
const out = run(`
|
||||
$t = @('To Be Filled By O.E.M.', 'Default string', 'System Serial Number', 'System Product Name', '0000000', ' ', 'None', 'Unknown', 'ASUS', 'ROG STRIX B560-F', 'PF2ABCDE', '0A1B')
|
||||
|
||||
Reference in new issue
Block a user