Host health from the Linux agent, agent self-update, and disk forecast in /metrics

- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates;
  SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox
- Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new
  "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series
- Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified
  download, atomic replace; GET /api/agent/report/update
- homelab_server_disk_days_to_full in /metrics
- Tests (server, real PowerShell against a fake server) and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 00:54:24 +02:00
1 parent 540ab704b7
commit 5dc6cea69c
38 files changed
+3884 -73

No files matched your search

+20
View File
@@ -10,6 +10,8 @@ import { getAlerts } from "./alerts.js";
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
import { getSourceStatus } from "./integrationHealthMonitor.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { computeSecretStatus } from "./secretStatus.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
@@ -80,8 +82,26 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
if (updates.security !== null) out.add("homelab_server_pending_security_updates", "gauge", "Of those, the ones that are security fixes.", updates.security, { server });
out.add("homelab_server_reboot_required", "gauge", "1 if a restart is due.", updates.rebootRequired ? 1 : 0, { server });
}
const health = parseHostHealth(s.hostHealth);
if (health?.failedUnits) out.add("homelab_server_failed_units", "gauge", "Services (systemd units) that have failed on the server.", health.failedUnits.length, { server });
for (const d of health?.smart ?? []) {
if (d.healthy !== null) out.add("homelab_server_disk_healthy", "gauge", "1 if the disk's own SMART self-assessment passes, 0 if it fails.", d.healthy ? 1 : 0, { server, device: d.device });
if (d.wearPercent !== null) out.add("homelab_server_disk_wear_percent", "gauge", "How much of an SSD's rated life is used.", d.wearPercent, { server, device: d.device });
if (d.temperatureC !== null) out.add("homelab_server_disk_temperature_celsius", "gauge", "A disk's temperature.", d.temperatureC, { server, device: d.device });
}
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a ZFS pool is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
out.add("homelab_server_agent_outdated", "gauge", "1 if the server's agent is older than the one this installation would install now.", isAgentOutdated(s, latest) ? 1 : 0, { server });
}
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
try {
for (const s of serverRows) {
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), 3650, 101)) {
out.add("homelab_server_disk_days_to_full", "gauge", "Days until a disk is full at its present rate of filling; only for disks with a steady climb.", f.daysToFull, { server: s.name, mount: f.mount });
}
}
} catch (err) {
console.error("[metrics] couldn't work out the disk forecasts:", err);
}
try {
const uptime = await getUptime(now, 30);
for (const s of serverRows) {