Host health from the Linux agent, agent self-update, and disk forecast in /metrics

- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates;
  SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox
- Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new
  "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series
- Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified
  download, atomic replace; GET /api/agent/report/update
- homelab_server_disk_days_to_full in /metrics
- Tests (server, real PowerShell against a fake server) and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 00:54:24 +02:00
1 parent 540ab704b7
commit 5dc6cea69c
38 files changed
+3884 -73

No files matched your search

+45
View File
@@ -4,6 +4,8 @@ import { db } from "../db/client.js";
import { hashToken } from "../services/tokens.js";
import { syncServerTasks } from "../services/taskSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { getSettings } from "../services/settingsStore.js";
import { latestAgentSha256, latestAgentVersions } from "../services/agentVersions.js";
export const agentReportRouter = Router();
@@ -31,7 +33,33 @@ const updatesSchema = z.object({
reboot_required: z.boolean().optional(),
});
// Failed services, disk health, ZFS pools. Each part is checked on its own and a bad one dropped, like the rest: it must never cost the report.
const healthSchema = z.object({
failed_units: z
.array(z.object({ unit: text(200), description: text(300).optional() }))
.transform(cap(100))
.optional()
.catch(undefined),
smart: z
.array(z.object({ device: text(100), model: text(200).optional(), healthy: z.boolean().nullable().optional(), temperature_c: z.number().nullable().optional(), wear_percent: z.number().nullable().optional() }))
.transform(cap(100))
.optional()
.catch(undefined),
zfs: z
.array(z.object({ name: text(200), health: text(40), capacity_percent: z.number().nullable().optional() }))
.transform(cap(100))
.optional()
.catch(undefined),
});
const systemSchema = z.object({
health: z
.unknown()
.optional()
.transform((h) => {
const parsed = healthSchema.safeParse(h);
return parsed.success && Object.values(parsed.data).some((v) => v !== undefined) ? parsed.data : undefined;
}),
updates: z
.unknown()
.optional()
@@ -105,6 +133,23 @@ async function authenticateAgent(req: Request, res: Response, next: NextFunction
next();
}
/**
* "Should I replace myself?" — asked by an agent that has been told to keep itself up to date, before it reports. Says yes only when an
* administrator has switched automatic updates on and this agent is behind the script this app serves, and gives the script's SHA-256 so
* the agent can check what it downloads. The agent says which version it is; failing that, the last one it reported counts.
*/
agentReportRouter.get("/update", asyncHandler(authenticateAgent), asyncHandler(async (req, res) => {
const server = res.locals.agentServer as { osType: string; agentVersion: number | null };
const os = server.osType === "windows" ? "windows" : "linux";
const claimed = Number(req.query.version);
const running = Number.isInteger(claimed) && claimed >= 1 && claimed <= 100_000 ? claimed : (server.agentVersion ?? 1);
const latest = latestAgentVersions()[os];
const sha256 = latestAgentSha256(os);
const { agents } = await getSettings();
const update = agents.autoUpdate && latest !== null && sha256 !== null && running < latest;
res.json({ update, version: update ? latest : null, sha256: update ? sha256 : null });
}));
agentReportRouter.post("/", asyncHandler(authenticateAgent), express.json({ limit: AGENT_BODY_LIMIT }), asyncHandler(async (req, res) => {
const server = res.locals.agentServer as { id: number };
+1
View File
@@ -20,6 +20,7 @@ alertTicketsRouter.use(requireAuth, requireRole("admin"));
const CATEGORIES = [
{ value: "offline", label: "Server down" },
{ value: "disk", label: "Disk & storage" },
{ value: "services", label: "Failing services" },
{ value: "backup", label: "Backups" },
{ value: "updates", label: "Updates" },
{ value: "expiry", label: "Expiry" },
+1 -1
View File
@@ -15,7 +15,7 @@ runbooksRouter.use(requireAuth);
/** More than anyone needs for one thing; each is another note to keep true. */
const MAX_PER_TARGET = 20;
export const ALERT_CATEGORIES = ["offline", "disk", "backup", "updates", "expiry", "automation", "integration", "monitoring", "tickets"] as const;
export const ALERT_CATEGORIES = ["offline", "disk", "services", "backup", "updates", "expiry", "automation", "integration", "monitoring", "tickets"] as const;
const bodySchema = z.object({
title: z.string().trim().min(1).max(120),
+19 -5
View File
@@ -8,6 +8,7 @@ import { generateApiToken } from "../services/tokens.js";
import { recordAudit } from "../services/audit.js";
import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js";
import { parsePendingUpdates } from "../services/pendingUpdates.js";
import { parseHostHealth } from "../services/hostHealth.js";
import { latestAgentVersions } from "../services/agentVersions.js";
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
import { getServerActivity } from "../services/serverActivity.js";
@@ -19,14 +20,14 @@ import { serverPortsRouter } from "./serverPorts.js";
import { InvalidTagError, normalizeTags, parseTags } from "../services/serverTags.js";
import { buildServerSummary } from "../services/serverSummary.js";
import { activeSubjects } from "../services/maintenance.js";
import { getSettings } from "../services/settingsStore.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
export const serversRouter = Router();
/** A server row as the API returns it: no token hash, and tags as an array rather than the stored JSON. */
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null }>(row: T) {
const { apiTokenHash: _hash, tags, pendingUpdates, ...rest } = row;
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates) };
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null; hostHealth: string | null }>(row: T) {
const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, ...rest } = row;
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth) };
}
serversRouter.use(requireAuth);
@@ -43,11 +44,23 @@ serversRouter.get("/", asyncHandler(async (_req, res) => {
const rows = await db.query.servers.findMany({ orderBy: (s, { asc }) => [asc(s.name)] });
res.json({
servers: rows.map(publicServer),
// Whether agents that are behind replace themselves, so the page can show the switch.
agentAutoUpdate: (await getSettings()).agents.autoUpdate,
// What a new install would be, so the page can say which agents are behind.
agentLatest: latestAgentVersions(),
});
}));
// Whether agents that are behind download the newer script from here and replace themselves. An administrator's to switch.
serversRouter.put("/agent-update", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = z.object({ autoUpdate: z.boolean() }).safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body" });
const before = (await getSettings()).agents.autoUpdate;
await updateSettings({ agents: { autoUpdate: parsed.data.autoUpdate } });
await recordAudit({ actor: req.currentUser!, category: "settings", action: "agent_auto_update", targetType: "settings", detail: { from: before, to: parsed.data.autoUpdate } });
res.json({ autoUpdate: parsed.data.autoUpdate });
}));
serversRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = createServerSchema.safeParse(req.body);
if (!parsed.success) {
@@ -353,6 +366,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
listeningPorts: _listeningPorts,
lastPortScan: _lastPortScan,
pendingUpdates: rawUpdates,
hostHealth: rawHealth,
...serverOut
} = server;
@@ -366,7 +380,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
children: [...links2.entries()].filter(([, l]) => l.parentId === id).map(([childId, l]) => ({ id: childId, name: nameOf.get(childId) ?? "", auto: l.auto })).sort((a, b) => a.name.localeCompare(b.name)),
};
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates) }, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: parseHostHealth(rawHealth) }, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
}));
const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) });