Host health from the Linux agent, agent self-update, and disk forecast in /metrics

- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates;
  SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox
- Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new
  "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series
- Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified
  download, atomic replace; GET /api/agent/report/update
- homelab_server_disk_days_to_full in /metrics
- Tests (server, real PowerShell against a fake server) and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 00:54:24 +02:00
1 parent 540ab704b7
commit 5dc6cea69c
38 files changed
+3884 -73

No files matched your search

+13 -2
View File
@@ -529,6 +529,8 @@ export interface ServerRecord {
updatesPendingSince: string | null;
/** Which version of the agent script it runs, as the agent says; null from one that predates the number (version 1). */
agentVersion: number | null;
/** Failed services, disk health and ZFS pools, as the agent says; null until it says anything. */
hostHealth: HostHealth | null;
}
/** The agent version a new install would be, per system; null when this app can't tell. */
@@ -537,6 +539,14 @@ export interface AgentLatest {
windows: number | null;
}
/** What a Linux agent says about the machine itself. Each part is there only if the agent could find it out. */
export interface HostHealth {
failedUnits?: { unit: string; description: string }[];
smart?: { device: string; model: string; healthy: boolean | null; temperatureC: number | null; wearPercent: number | null }[];
zfs?: { name: string; health: string; capacityPercent: number | null }[];
reportedAt: string;
}
export interface PendingUpdates {
manager: string;
count: number;
@@ -727,7 +737,7 @@ export interface Runbook {
export type RunbookSummary = Pick<Runbook, "id" | "targetType" | "targetId" | "category" | "title">;
export type AlertSeverity = "critical" | "warning" | "info";
export type AlertCategory = "offline" | "disk" | "backup" | "updates" | "expiry" | "automation" | "integration" | "monitoring" | "tickets";
export type AlertCategory = "offline" | "disk" | "services" | "backup" | "updates" | "expiry" | "automation" | "integration" | "monitoring" | "tickets";
export interface AlertItem {
id: string;
@@ -1641,7 +1651,8 @@ export const api = {
clearCache: () => request<{ ok: true }>("/api/dns/cache/clear", { method: "POST" }),
},
servers: {
list: () => request<{ servers: ServerRecord[]; agentLatest: AgentLatest }>("/api/servers"),
list: () => request<{ servers: ServerRecord[]; agentLatest: AgentLatest; agentAutoUpdate: boolean }>("/api/servers"),
setAgentAutoUpdate: (autoUpdate: boolean) => request<{ autoUpdate: boolean }>("/api/servers/agent-update", { method: "PUT", body: JSON.stringify({ autoUpdate }) }),
create: (data: { name: string; hostname?: string; description?: string; osType?: "linux" | "windows" }) =>
request<{ server: ServerRecord; token: string }>("/api/servers", {
method: "POST",
+121
View File
@@ -0,0 +1,121 @@
import type { HostHealth } from "../api/client";
import TimeAgo from "./TimeAgo";
/** The degraded states of a ZFS pool are all "not fine", but only ONLINE is quite fine. */
const poolFine = (health: string) => health.toUpperCase() === "ONLINE";
/**
* What the agent says about the machine itself: services that have failed, whether each disk's SMART check passes and how worn it is, and
* whether each ZFS pool is online. Each part appears only if the agent reported it (no smartctl or no ZFS on the host means no part),
* and the card is left out when it has nothing to say.
*/
export default function HostHealthCard({ health }: { health: HostHealth | null }) {
if (!health) return null;
const { failedUnits, smart, zfs } = health;
const hasSmart = !!smart && smart.length > 0;
const hasZfs = !!zfs && zfs.length > 0;
if (failedUnits === undefined && !hasSmart && !hasZfs) return null;
return (
<div className="card mb-3">
<div className="card-header">
<h3 className="card-title">Host health</h3>
{health.reportedAt && (
<div className="card-actions text-secondary small">
reported <TimeAgo value={health.reportedAt} />
</div>
)}
</div>
<div className="card-body">
{failedUnits !== undefined && (
<div className={hasSmart || hasZfs ? "mb-3" : ""}>
<div className="text-secondary small mb-1">Services</div>
{failedUnits.length === 0 ? (
<span className="badge bg-green-lt text-green">none have failed</span>
) : (
<ul className="list-unstyled mb-0">
{failedUnits.map((u) => (
<li key={u.unit}>
<span className="badge bg-red-lt text-red me-2">failed</span>
<code>{u.unit}</code>
{u.description && <span className="text-secondary small ms-2">{u.description}</span>}
</li>
))}
</ul>
)}
</div>
)}
{hasSmart && (
<div className={hasZfs ? "mb-3" : ""}>
<div className="text-secondary small mb-1">Disks (SMART)</div>
<div className="table-responsive">
<table className="table table-sm table-vcenter mb-0">
<thead>
<tr>
<th>Disk</th>
<th>Model</th>
<th>Verdict</th>
<th>Temperature</th>
<th>Life used</th>
</tr>
</thead>
<tbody>
{smart!.map((d) => (
<tr key={d.device}>
<td>
<code>{d.device}</code>
</td>
<td className="text-secondary">{d.model || "—"}</td>
<td>
{d.healthy === null ? (
<span className="text-secondary">—</span>
) : d.healthy ? (
<span className="badge bg-green-lt text-green">passes</span>
) : (
<span className="badge bg-red-lt text-red">failing</span>
)}
</td>
<td>{d.temperatureC === null ? "—" : `${Math.round(d.temperatureC)} °C`}</td>
<td>{d.wearPercent === null ? "—" : `${Math.round(d.wearPercent)}%`}</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
)}
{hasZfs && (
<div>
<div className="text-secondary small mb-1">ZFS pools</div>
<div className="table-responsive">
<table className="table table-sm table-vcenter mb-0">
<thead>
<tr>
<th>Pool</th>
<th>State</th>
<th>Used</th>
</tr>
</thead>
<tbody>
{zfs!.map((p) => (
<tr key={p.name}>
<td>
<code>{p.name}</code>
</td>
<td>
<span className={`badge ${poolFine(p.health) ? "bg-green-lt text-green" : "bg-red-lt text-red"}`}>{p.health}</span>
</td>
<td>{p.capacityPercent === null ? "—" : `${Math.round(p.capacityPercent)}%`}</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
)}
</div>
</div>
);
}
+2 -1
View File
@@ -10,7 +10,8 @@ import TimeAgo from "./TimeAgo";
export const CATEGORY_WORDS: Record<AlertCategory, string> = {
offline: "a server that stopped reporting",
disk: "a disk filling up",
disk: "a disk filling up or failing",
services: "a failed service",
backup: "a backup problem",
updates: "updates waiting",
expiry: "something expiring",
+1
View File
@@ -24,6 +24,7 @@ const SEVERITY: Record<AlertSeverity, { label: string; badge: string; rank: numb
const CATEGORY_LABELS: Record<AlertCategory, string> = {
offline: "Server down",
disk: "Disk & storage",
services: "Failing services",
backup: "Backups",
updates: "Updates",
expiry: "Expiry",
+3 -1
View File
@@ -18,6 +18,7 @@ import ServerActivity from "../components/ServerActivity";
import TimeAgo from "../components/TimeAgo";
import UptimeRibbon from "../components/UptimeRibbon";
import RunsOn from "../components/RunsOn";
import HostHealthCard from "../components/HostHealthCard";
import RunPlaybook from "../components/RunPlaybook";
import RunbookPanel from "../components/RunbookPanel";
import UpdatesBadge from "../components/UpdatesBadge";
@@ -783,7 +784,7 @@ export default function ServerDetail({ user }: { user: CurrentUser }) {
<dd className="col-7">
{agentVersionWords(server.agentVersion)}{" "}
{agentState(server, detail.agentLatest) === "behind" ? (
<span className="badge bg-yellow-lt text-yellow" title="Run its install command again to update it (Servers → Manage servers).">
<span className="badge bg-yellow-lt text-yellow" title="Run its install command again to update it (Servers → Manage servers), or switch on automatic agent updates on the Servers page.">
update available
</span>
) : (
@@ -807,6 +808,7 @@ export default function ServerDetail({ user }: { user: CurrentUser }) {
</div>
</div>
<HostHealthCard health={server.hostHealth} />
{hardware.source === "agent" && <ServerTrends serverId={serverId} />}
{(user.role === "admin" || user.role === "operator") && <ServerActivity serverId={serverId} />}
<div className="card mb-3">
+30
View File
@@ -16,6 +16,7 @@ import { tagBadge, useTagColors } from "../utils/tags";
import { confirmDialog } from "../utils/dialogs";
import { ErrorNotice } from "../components/Notice";
import { toast } from "../utils/toast";
import { readableError } from "../utils/errors";
import { SearchBox } from "../components/TableToolbar";
import { useTableSearch } from "../hooks/useTableSearch";
import SaveViewButton from "../components/SaveViewButton";
@@ -36,6 +37,8 @@ export default function Servers({ user }: { user: CurrentUser }) {
// Until the first answer arrives there is nothing to say about "no servers", so show the page's shape instead of the empty message.
const [loaded, setLoaded] = useState(false);
const [agentLatest, setAgentLatest] = useState<AgentLatest | null>(null);
// Whether agents that are behind replace themselves (an administrator's switch).
const [autoUpdate, setAutoUpdate] = useState<boolean | null>(null);
const [error, setError] = useState<string | null>(null);
const [managingServers, setManagingServers] = useState(false);
const [integrationColors, setIntegrationColors] = useState<Record<string, string>>({});
@@ -76,6 +79,7 @@ export default function Servers({ user }: { user: CurrentUser }) {
.then((res) => {
setServers(res.servers);
setAgentLatest(res.agentLatest);
setAutoUpdate(res.agentAutoUpdate);
})
.catch((err) => setError(String(err)))
.finally(() => setLoaded(true));
@@ -85,6 +89,17 @@ export default function Servers({ user }: { user: CurrentUser }) {
loadServers();
}, [loadServers]);
async function changeAutoUpdate(value: boolean) {
setAutoUpdate(value);
try {
await api.servers.setAgentAutoUpdate(value);
toast.success(value ? "Agents that are behind will update themselves at their next run." : "Agents will no longer update themselves.");
} catch (err) {
setAutoUpdate(!value);
toast.error(readableError(err));
}
}
async function createServer(e: React.FormEvent) {
e.preventDefault();
setError(null);
@@ -245,6 +260,21 @@ export default function Servers({ user }: { user: CurrentUser }) {
</div>
<ErrorNotice error={error} />
{isAdmin && autoUpdate !== null && servers.length > 0 && (
<div className="card mb-3">
<div className="card-body py-2">
<label className="form-check form-switch mb-0">
<input className="form-check-input" type="checkbox" checked={autoUpdate} onChange={(e) => void changeAutoUpdate(e.target.checked)} />
<span className="form-check-label">Update agents automatically</span>
</label>
<div className="text-secondary small mt-1">
An agent that is behind this app downloads the newer script from here, checks it against a checksum the server gives, and replaces itself at its next run. Off by default. An agent needs the
update code to use it (Linux version 4 or later, Windows 3 or later); an older one needs its install command run once (Manage servers).
</div>
</div>
</div>
)}
{managingServers ? (
<>
<div className="card mb-3">