Host health from the Linux agent, agent self-update, and disk forecast in /metrics
- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates; SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox - Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series - Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified download, atomic replace; GET /api/agent/report/update - homelab_server_disk_days_to_full in /metrics - Tests (server, real PowerShell against a fake server) and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
540ab704b7
commit
5dc6cea69c
38 files changed
+3884
-73
No files matched your search
@@ -6,8 +6,11 @@
|
||||
*
|
||||
* 1 — before the number was reported (so an agent that says nothing is version 1)
|
||||
* 2 — reports pending updates, and its own version
|
||||
* 3 — Linux: the package manager is asked outside the systemd sandbox, so dnf and zypper can answer too (Windows stays at 2)
|
||||
* 3 — Linux: the package manager is asked outside the systemd sandbox, so dnf and zypper can answer too.
|
||||
* Windows: can replace itself when an administrator has switched automatic updates on.
|
||||
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
@@ -51,6 +54,18 @@ export function latestAgentVersions(dir: string = AGENT_DIR): LatestAgentVersion
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* The SHA-256 of the script this app serves for an OS, as hex — what an agent that updates itself checks the download against, so a
|
||||
* truncated or tampered-with transfer is never installed. Null if the script can't be read.
|
||||
*/
|
||||
export function latestAgentSha256(os: AgentOs, dir: string = AGENT_DIR): string | null {
|
||||
try {
|
||||
return createHash("sha256").update(readFileSync(join(dir, FILES[os]))).digest("hex");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a server's agent is behind. Only a server that reports through an agent can be: one that never has (a Proxmox guest tracked
|
||||
* through its host, say) has none to update. An agent that reports no version is version 1.
|
||||
|
||||
@@ -126,6 +126,7 @@ export function planTicketRun(input: {
|
||||
const CATEGORY_LABELS: Record<AlertCategory, string> = {
|
||||
offline: "Server down",
|
||||
disk: "Disk & storage",
|
||||
services: "Failing services",
|
||||
backup: "Backups",
|
||||
updates: "Updates",
|
||||
expiry: "Expiry",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
export type AlertSeverity = "critical" | "warning" | "info";
|
||||
|
||||
/** What kind of problem — drives the filter on the Alerts page. */
|
||||
export type AlertCategory = "offline" | "disk" | "backup" | "updates" | "expiry" | "automation" | "integration" | "monitoring" | "tickets";
|
||||
export type AlertCategory = "offline" | "disk" | "services" | "backup" | "updates" | "expiry" | "automation" | "integration" | "monitoring" | "tickets";
|
||||
|
||||
export interface Alert {
|
||||
/** Stable, so the page can key rows on it. */
|
||||
|
||||
@@ -68,8 +68,8 @@ const SEVERITY_ORDER: Record<AlertSeverity, number> = { critical: 0, warning: 1,
|
||||
|
||||
/** Which kind of source, and which one, a health/automation condition key belongs to. */
|
||||
function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } {
|
||||
let m = /^(?:offline|disk):server:(\d+)/.exec(key);
|
||||
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : "disk" };
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)/.exec(key);
|
||||
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : "disk" };
|
||||
m = /^disk:proxmox:(\d+):/.exec(key);
|
||||
if (m) return { type: "proxmox", id: Number(m[1]), category: "disk" };
|
||||
m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key);
|
||||
|
||||
@@ -9,6 +9,7 @@ import { recordAudit } from "./audit.js";
|
||||
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
|
||||
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
|
||||
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
|
||||
import { hostHealthConditions, parseHostHealth, type HostHealth } from "./hostHealth.js";
|
||||
|
||||
const STATE_FLAG = "healthActiveConditions";
|
||||
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
|
||||
@@ -32,6 +33,8 @@ export interface ServerSnapshot {
|
||||
name: string;
|
||||
lastSeenAt: string | null;
|
||||
disks: { mount: string; sizeBytes: number; usedBytes: number }[];
|
||||
/** What the agent says about failed services, disk health and ZFS pools, if it says anything. */
|
||||
health?: HostHealth | null;
|
||||
}
|
||||
export interface ProxmoxSnapshot {
|
||||
integrationId: number;
|
||||
@@ -115,6 +118,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
}
|
||||
// A server that never reported has no agent to go quiet; and an offline one's disk figures are stale, so neither is judged on disks.
|
||||
if (s.lastSeenAt && !offline) {
|
||||
for (const c of hostHealthConditions(s.id, s.name, s.health ?? null)) out.push({ key: c.key, source: "server", message: c.message, severity: c.severity });
|
||||
for (const d of s.disks) {
|
||||
const p = usage(d.usedBytes, d.sizeBytes);
|
||||
if (p !== null && p >= limit) {
|
||||
@@ -296,7 +300,7 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
} catch {
|
||||
// a malformed report shouldn't stop every other server being checked
|
||||
}
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks });
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: parseHostHealth(s.hostHealth) });
|
||||
}
|
||||
|
||||
const rows = await db
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
/**
|
||||
* What a Linux agent can say about the machine's own health, beyond its figures: services that have failed (systemd), whether each disk's
|
||||
* SMART self-assessment passes and how worn it is, and whether each ZFS pool is online. Each part is optional — an agent reports only what
|
||||
* it could find out (no smartctl, no ZFS, no systemd) — and a part that is absent from a report keeps what was last said, so an agent that
|
||||
* skips one never turns an old problem into "all well".
|
||||
*/
|
||||
|
||||
export interface FailedUnit {
|
||||
unit: string;
|
||||
description: string;
|
||||
}
|
||||
export interface SmartDisk {
|
||||
device: string;
|
||||
model: string;
|
||||
/** The drive's own overall verdict: true passes, false fails, null if it didn't give one. */
|
||||
healthy: boolean | null;
|
||||
temperatureC: number | null;
|
||||
/** NVMe and some SSDs say how much of their rated life is used, 0–100 (and past it). */
|
||||
wearPercent: number | null;
|
||||
}
|
||||
export interface ZfsPool {
|
||||
name: string;
|
||||
/** ONLINE, DEGRADED, FAULTED, UNAVAIL, SUSPENDED … as zpool says it. */
|
||||
health: string;
|
||||
capacityPercent: number | null;
|
||||
}
|
||||
|
||||
export interface HostHealth {
|
||||
failedUnits?: FailedUnit[];
|
||||
smart?: SmartDisk[];
|
||||
zfs?: ZfsPool[];
|
||||
/** ISO — when the agent last told us anything here. */
|
||||
reportedAt: string;
|
||||
}
|
||||
|
||||
/** As it arrives from an agent. */
|
||||
export interface IncomingHostHealth {
|
||||
failed_units?: { unit: string; description?: string }[];
|
||||
smart?: { device: string; model?: string; healthy?: boolean | null; temperature_c?: number | null; wear_percent?: number | null }[];
|
||||
zfs?: { name: string; health: string; capacity_percent?: number | null }[];
|
||||
}
|
||||
|
||||
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||||
|
||||
export function parseHostHealth(stored: string | null): HostHealth | null {
|
||||
if (!stored) return null;
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
if (!v || typeof v !== "object") return null;
|
||||
const out: HostHealth = { reportedAt: typeof v.reportedAt === "string" ? v.reportedAt : "" };
|
||||
if (Array.isArray(v.failedUnits)) out.failedUnits = v.failedUnits.filter((u: any) => u && typeof u.unit === "string").map((u: any) => ({ unit: u.unit, description: typeof u.description === "string" ? u.description : "" }));
|
||||
if (Array.isArray(v.smart)) {
|
||||
out.smart = v.smart
|
||||
.filter((d: any) => d && typeof d.device === "string")
|
||||
.map((d: any) => ({ device: d.device, model: typeof d.model === "string" ? d.model : "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperatureC), wearPercent: num(d.wearPercent) }));
|
||||
}
|
||||
if (Array.isArray(v.zfs)) out.zfs = v.zfs.filter((p: any) => p && typeof p.name === "string" && typeof p.health === "string").map((p: any) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacityPercent) }));
|
||||
return out;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The text to store after a report: the parts the agent sent replace what was there, the parts it didn't send stay. */
|
||||
export function nextHostHealth(previous: string | null, incoming: IncomingHostHealth, now: string): string | null {
|
||||
const keep = parseHostHealth(previous);
|
||||
const next: HostHealth = { ...(keep ?? {}), reportedAt: now };
|
||||
let any = false;
|
||||
if (incoming.failed_units) {
|
||||
any = true;
|
||||
next.failedUnits = incoming.failed_units.map((u) => ({ unit: u.unit, description: u.description ?? "" }));
|
||||
}
|
||||
if (incoming.smart) {
|
||||
any = true;
|
||||
next.smart = incoming.smart.map((d) => ({ device: d.device, model: d.model ?? "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperature_c), wearPercent: num(d.wear_percent) }));
|
||||
}
|
||||
if (incoming.zfs) {
|
||||
any = true;
|
||||
next.zfs = incoming.zfs.map((p) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacity_percent) }));
|
||||
}
|
||||
return any ? JSON.stringify(next) : previous;
|
||||
}
|
||||
|
||||
/** A pool in any state but ONLINE has lost something (DEGRADED has lost redundancy, the rest are worse). */
|
||||
export const poolIsFine = (p: ZfsPool) => p.health.toUpperCase() === "ONLINE";
|
||||
|
||||
/** Wear at which a drive is worth replacing soon, in percent of rated life used. */
|
||||
export const WEAR_WARNING_PERCENT = 90;
|
||||
|
||||
export interface HostHealthCondition {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "critical" | "warning";
|
||||
}
|
||||
|
||||
const listOf = (names: string[], max = 4) => (names.length <= max ? names.join(", ") : `${names.slice(0, max).join(", ")} and ${names.length - max} more`);
|
||||
|
||||
/** The problems a server's health report amounts to. Pure, so the rules can be tested exactly. */
|
||||
export function hostHealthConditions(serverId: number, serverName: string, health: HostHealth | null): HostHealthCondition[] {
|
||||
if (!health) return [];
|
||||
const out: HostHealthCondition[] = [];
|
||||
const failed = health.failedUnits ?? [];
|
||||
if (failed.length > 0) {
|
||||
out.push({
|
||||
key: `failed-units:server:${serverId}`,
|
||||
severity: "warning",
|
||||
message: `${serverName}: ${failed.length === 1 ? "1 service has" : `${failed.length} services have`} failed — ${listOf(failed.map((u) => u.unit))}`,
|
||||
});
|
||||
}
|
||||
for (const d of health.smart ?? []) {
|
||||
const label = `${d.device}${d.model ? ` (${d.model})` : ""}`;
|
||||
if (d.healthy === false) {
|
||||
out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "critical", message: `${serverName}: the disk ${label} says it is failing (SMART self-assessment)` });
|
||||
} else if (d.wearPercent !== null && d.wearPercent >= WEAR_WARNING_PERCENT) {
|
||||
out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "warning", message: `${serverName}: the disk ${label} has used ${Math.round(d.wearPercent)}% of its rated life` });
|
||||
}
|
||||
}
|
||||
for (const p of health.zfs ?? []) {
|
||||
if (!poolIsFine(p)) {
|
||||
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: ZFS pool ${p.name} is ${p.health}` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -23,7 +23,7 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
|
||||
* so conditions persisted by an earlier version still map correctly.
|
||||
*/
|
||||
export function subjectOfConditionKey(key: string): string | null {
|
||||
let m = /^(?:offline|disk):server:(\d+)(?::|$)/.exec(key);
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)(?::|$)/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^disk:proxmox:(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
|
||||
@@ -10,6 +10,8 @@ import { getAlerts } from "./alerts.js";
|
||||
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
|
||||
import { getSourceStatus } from "./integrationHealthMonitor.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
@@ -80,8 +82,26 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
if (updates.security !== null) out.add("homelab_server_pending_security_updates", "gauge", "Of those, the ones that are security fixes.", updates.security, { server });
|
||||
out.add("homelab_server_reboot_required", "gauge", "1 if a restart is due.", updates.rebootRequired ? 1 : 0, { server });
|
||||
}
|
||||
const health = parseHostHealth(s.hostHealth);
|
||||
if (health?.failedUnits) out.add("homelab_server_failed_units", "gauge", "Services (systemd units) that have failed on the server.", health.failedUnits.length, { server });
|
||||
for (const d of health?.smart ?? []) {
|
||||
if (d.healthy !== null) out.add("homelab_server_disk_healthy", "gauge", "1 if the disk's own SMART self-assessment passes, 0 if it fails.", d.healthy ? 1 : 0, { server, device: d.device });
|
||||
if (d.wearPercent !== null) out.add("homelab_server_disk_wear_percent", "gauge", "How much of an SSD's rated life is used.", d.wearPercent, { server, device: d.device });
|
||||
if (d.temperatureC !== null) out.add("homelab_server_disk_temperature_celsius", "gauge", "A disk's temperature.", d.temperatureC, { server, device: d.device });
|
||||
}
|
||||
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a ZFS pool is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
|
||||
out.add("homelab_server_agent_outdated", "gauge", "1 if the server's agent is older than the one this installation would install now.", isAgentOutdated(s, latest) ? 1 : 0, { server });
|
||||
}
|
||||
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
|
||||
try {
|
||||
for (const s of serverRows) {
|
||||
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), 3650, 101)) {
|
||||
out.add("homelab_server_disk_days_to_full", "gauge", "Days until a disk is full at its present rate of filling; only for disks with a steady climb.", f.daysToFull, { server: s.name, mount: f.mount });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[metrics] couldn't work out the disk forecasts:", err);
|
||||
}
|
||||
try {
|
||||
const uptime = await getUptime(now, 30);
|
||||
for (const s of serverRows) {
|
||||
|
||||
@@ -54,6 +54,8 @@ export function describeAudit(category: string, action: string, detail: Record<s
|
||||
return { title: "A disk got nearly full", tone: "bad", kind: "health", detail: message };
|
||||
case "disk_ok":
|
||||
return { title: "A disk is no longer nearly full", tone: "good", kind: "health", detail: null };
|
||||
case "agent_updated":
|
||||
return { title: `Agent updated to version ${detail.to ?? "?"}`, tone: "good", kind: "change", detail: detail.from !== undefined ? `from version ${detail.from}` : null };
|
||||
case "create":
|
||||
return { title: "Server added", tone: "neutral", kind: "change", detail: null };
|
||||
case "update":
|
||||
|
||||
@@ -119,6 +119,12 @@ export interface SignInSettings {
|
||||
otherwise: "viewer" | "keep";
|
||||
}
|
||||
|
||||
/** How the agents out on the servers are kept up to date. Off until an administrator turns it on. */
|
||||
export interface AgentSettings {
|
||||
/** An agent that is behind downloads the newer script from this app, checks it and replaces itself. */
|
||||
autoUpdate: boolean;
|
||||
}
|
||||
|
||||
/** What the inventory needs to work out a running cost. Without a price per kWh it shows draw but no cost. */
|
||||
export interface InventorySettings {
|
||||
/** A short name shown after amounts ("SEK", "kr", "$"); may be blank. */
|
||||
@@ -180,6 +186,7 @@ export interface AppSettings {
|
||||
digest: DigestSettings;
|
||||
signIn: SignInSettings;
|
||||
inventory: InventorySettings;
|
||||
agents: AgentSettings;
|
||||
consistency: ConsistencySettings;
|
||||
nameGenerator: NameGeneratorSettings;
|
||||
alertTickets: AlertTicketSettings;
|
||||
@@ -214,6 +221,7 @@ const DEFAULTS: AppSettings = {
|
||||
quietHours: { enabled: false, start: "22:00", end: "07:00" },
|
||||
digest: { enabled: false, day: 1, time: "08:00" },
|
||||
inventory: { currency: "", pricePerKwh: null },
|
||||
agents: { autoUpdate: false },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
|
||||
@@ -3,6 +3,8 @@ import { db } from "../db/client.js";
|
||||
import { scheduledTasks, servers } from "../db/schema.js";
|
||||
import { recordServerMetrics } from "./serverMetrics.js";
|
||||
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
|
||||
import { nextHostHealth, type IncomingHostHealth } from "./hostHealth.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
|
||||
export interface IncomingTask {
|
||||
scheduleType: "cron" | "systemd_timer" | "windows_task";
|
||||
@@ -23,6 +25,8 @@ export interface IncomingSystemInfo {
|
||||
listening_ports?: { protocol: "tcp" | "udp"; port: number; address: string; process?: string }[];
|
||||
/** Updates waiting to be installed. Absent from an agent that doesn't report them (or couldn't find out this time). */
|
||||
updates?: IncomingUpdates;
|
||||
/** Failed services, disk health, ZFS pools — each part present only if the agent could find it out. */
|
||||
health?: IncomingHostHealth;
|
||||
}
|
||||
|
||||
export interface AgentReport {
|
||||
@@ -103,13 +107,14 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
}
|
||||
|
||||
const system = report.system;
|
||||
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince }).from(servers).where(eq(servers.id, serverId));
|
||||
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, hostHealth: servers.hostHealth, agentVersion: servers.agentVersion }).from(servers).where(eq(servers.id, serverId));
|
||||
await db
|
||||
.update(servers)
|
||||
.set({
|
||||
lastSeenAt: now,
|
||||
// Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before.
|
||||
...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}),
|
||||
...(system?.health && before ? { hostHealth: nextHostHealth(before.hostHealth, system.health, now) } : {}),
|
||||
...(report.hostname ? { hostname: report.hostname } : {}),
|
||||
...(report.agentVersion !== undefined ? { agentVersion: report.agentVersion } : {}),
|
||||
// An agent knows what it runs on better than whoever registered the server did.
|
||||
@@ -132,6 +137,15 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
})
|
||||
.where(eq(servers.id, serverId));
|
||||
|
||||
// An agent that now says a higher version than it did has been updated (by hand, or by itself): worth a line in the server's history.
|
||||
if (before && report.agentVersion !== undefined && before.agentVersion !== null && report.agentVersion > before.agentVersion) {
|
||||
try {
|
||||
await recordAudit({ category: "server", action: "agent_updated", targetType: "server", targetId: serverId, detail: { from: before.agentVersion, to: report.agentVersion } });
|
||||
} catch (err) {
|
||||
console.error("[agent] couldn't record the agent update:", err);
|
||||
}
|
||||
}
|
||||
|
||||
// Keep a little history of the figures too. A failure here must not lose the report itself, which is already saved.
|
||||
try {
|
||||
await recordServerMetrics(serverId, system, new Date(now));
|
||||
|
||||
Reference in new issue
Block a user