Host health from the Linux agent, agent self-update, and disk forecast in /metrics
- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates; SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox - Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series - Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified download, atomic replace; GET /api/agent/report/update - homelab_server_disk_days_to_full in /metrics - Tests (server, real PowerShell against a fake server) and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
540ab704b7
commit
5dc6cea69c
38 files changed
+3884
-73
No files matched your search
+13
-2
@@ -529,6 +529,8 @@ export interface ServerRecord {
|
||||
updatesPendingSince: string | null;
|
||||
/** Which version of the agent script it runs, as the agent says; null from one that predates the number (version 1). */
|
||||
agentVersion: number | null;
|
||||
/** Failed services, disk health and ZFS pools, as the agent says; null until it says anything. */
|
||||
hostHealth: HostHealth | null;
|
||||
}
|
||||
|
||||
/** The agent version a new install would be, per system; null when this app can't tell. */
|
||||
@@ -537,6 +539,14 @@ export interface AgentLatest {
|
||||
windows: number | null;
|
||||
}
|
||||
|
||||
/** What a Linux agent says about the machine itself. Each part is there only if the agent could find it out. */
|
||||
export interface HostHealth {
|
||||
failedUnits?: { unit: string; description: string }[];
|
||||
smart?: { device: string; model: string; healthy: boolean | null; temperatureC: number | null; wearPercent: number | null }[];
|
||||
zfs?: { name: string; health: string; capacityPercent: number | null }[];
|
||||
reportedAt: string;
|
||||
}
|
||||
|
||||
export interface PendingUpdates {
|
||||
manager: string;
|
||||
count: number;
|
||||
@@ -727,7 +737,7 @@ export interface Runbook {
|
||||
export type RunbookSummary = Pick<Runbook, "id" | "targetType" | "targetId" | "category" | "title">;
|
||||
|
||||
export type AlertSeverity = "critical" | "warning" | "info";
|
||||
export type AlertCategory = "offline" | "disk" | "backup" | "updates" | "expiry" | "automation" | "integration" | "monitoring" | "tickets";
|
||||
export type AlertCategory = "offline" | "disk" | "services" | "backup" | "updates" | "expiry" | "automation" | "integration" | "monitoring" | "tickets";
|
||||
|
||||
export interface AlertItem {
|
||||
id: string;
|
||||
@@ -1641,7 +1651,8 @@ export const api = {
|
||||
clearCache: () => request<{ ok: true }>("/api/dns/cache/clear", { method: "POST" }),
|
||||
},
|
||||
servers: {
|
||||
list: () => request<{ servers: ServerRecord[]; agentLatest: AgentLatest }>("/api/servers"),
|
||||
list: () => request<{ servers: ServerRecord[]; agentLatest: AgentLatest; agentAutoUpdate: boolean }>("/api/servers"),
|
||||
setAgentAutoUpdate: (autoUpdate: boolean) => request<{ autoUpdate: boolean }>("/api/servers/agent-update", { method: "PUT", body: JSON.stringify({ autoUpdate }) }),
|
||||
create: (data: { name: string; hostname?: string; description?: string; osType?: "linux" | "windows" }) =>
|
||||
request<{ server: ServerRecord; token: string }>("/api/servers", {
|
||||
method: "POST",
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
import type { HostHealth } from "../api/client";
|
||||
import TimeAgo from "./TimeAgo";
|
||||
|
||||
/** The degraded states of a ZFS pool are all "not fine", but only ONLINE is quite fine. */
|
||||
const poolFine = (health: string) => health.toUpperCase() === "ONLINE";
|
||||
|
||||
/**
|
||||
* What the agent says about the machine itself: services that have failed, whether each disk's SMART check passes and how worn it is, and
|
||||
* whether each ZFS pool is online. Each part appears only if the agent reported it (no smartctl or no ZFS on the host means no part),
|
||||
* and the card is left out when it has nothing to say.
|
||||
*/
|
||||
export default function HostHealthCard({ health }: { health: HostHealth | null }) {
|
||||
if (!health) return null;
|
||||
const { failedUnits, smart, zfs } = health;
|
||||
const hasSmart = !!smart && smart.length > 0;
|
||||
const hasZfs = !!zfs && zfs.length > 0;
|
||||
if (failedUnits === undefined && !hasSmart && !hasZfs) return null;
|
||||
|
||||
return (
|
||||
<div className="card mb-3">
|
||||
<div className="card-header">
|
||||
<h3 className="card-title">Host health</h3>
|
||||
{health.reportedAt && (
|
||||
<div className="card-actions text-secondary small">
|
||||
reported <TimeAgo value={health.reportedAt} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="card-body">
|
||||
{failedUnits !== undefined && (
|
||||
<div className={hasSmart || hasZfs ? "mb-3" : ""}>
|
||||
<div className="text-secondary small mb-1">Services</div>
|
||||
{failedUnits.length === 0 ? (
|
||||
<span className="badge bg-green-lt text-green">none have failed</span>
|
||||
) : (
|
||||
<ul className="list-unstyled mb-0">
|
||||
{failedUnits.map((u) => (
|
||||
<li key={u.unit}>
|
||||
<span className="badge bg-red-lt text-red me-2">failed</span>
|
||||
<code>{u.unit}</code>
|
||||
{u.description && <span className="text-secondary small ms-2">{u.description}</span>}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{hasSmart && (
|
||||
<div className={hasZfs ? "mb-3" : ""}>
|
||||
<div className="text-secondary small mb-1">Disks (SMART)</div>
|
||||
<div className="table-responsive">
|
||||
<table className="table table-sm table-vcenter mb-0">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Disk</th>
|
||||
<th>Model</th>
|
||||
<th>Verdict</th>
|
||||
<th>Temperature</th>
|
||||
<th>Life used</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{smart!.map((d) => (
|
||||
<tr key={d.device}>
|
||||
<td>
|
||||
<code>{d.device}</code>
|
||||
</td>
|
||||
<td className="text-secondary">{d.model || "—"}</td>
|
||||
<td>
|
||||
{d.healthy === null ? (
|
||||
<span className="text-secondary">—</span>
|
||||
) : d.healthy ? (
|
||||
<span className="badge bg-green-lt text-green">passes</span>
|
||||
) : (
|
||||
<span className="badge bg-red-lt text-red">failing</span>
|
||||
)}
|
||||
</td>
|
||||
<td>{d.temperatureC === null ? "—" : `${Math.round(d.temperatureC)} °C`}</td>
|
||||
<td>{d.wearPercent === null ? "—" : `${Math.round(d.wearPercent)}%`}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{hasZfs && (
|
||||
<div>
|
||||
<div className="text-secondary small mb-1">ZFS pools</div>
|
||||
<div className="table-responsive">
|
||||
<table className="table table-sm table-vcenter mb-0">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Pool</th>
|
||||
<th>State</th>
|
||||
<th>Used</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{zfs!.map((p) => (
|
||||
<tr key={p.name}>
|
||||
<td>
|
||||
<code>{p.name}</code>
|
||||
</td>
|
||||
<td>
|
||||
<span className={`badge ${poolFine(p.health) ? "bg-green-lt text-green" : "bg-red-lt text-red"}`}>{p.health}</span>
|
||||
</td>
|
||||
<td>{p.capacityPercent === null ? "—" : `${Math.round(p.capacityPercent)}%`}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -10,7 +10,8 @@ import TimeAgo from "./TimeAgo";
|
||||
|
||||
export const CATEGORY_WORDS: Record<AlertCategory, string> = {
|
||||
offline: "a server that stopped reporting",
|
||||
disk: "a disk filling up",
|
||||
disk: "a disk filling up or failing",
|
||||
services: "a failed service",
|
||||
backup: "a backup problem",
|
||||
updates: "updates waiting",
|
||||
expiry: "something expiring",
|
||||
|
||||
@@ -24,6 +24,7 @@ const SEVERITY: Record<AlertSeverity, { label: string; badge: string; rank: numb
|
||||
const CATEGORY_LABELS: Record<AlertCategory, string> = {
|
||||
offline: "Server down",
|
||||
disk: "Disk & storage",
|
||||
services: "Failing services",
|
||||
backup: "Backups",
|
||||
updates: "Updates",
|
||||
expiry: "Expiry",
|
||||
|
||||
@@ -18,6 +18,7 @@ import ServerActivity from "../components/ServerActivity";
|
||||
import TimeAgo from "../components/TimeAgo";
|
||||
import UptimeRibbon from "../components/UptimeRibbon";
|
||||
import RunsOn from "../components/RunsOn";
|
||||
import HostHealthCard from "../components/HostHealthCard";
|
||||
import RunPlaybook from "../components/RunPlaybook";
|
||||
import RunbookPanel from "../components/RunbookPanel";
|
||||
import UpdatesBadge from "../components/UpdatesBadge";
|
||||
@@ -783,7 +784,7 @@ export default function ServerDetail({ user }: { user: CurrentUser }) {
|
||||
<dd className="col-7">
|
||||
{agentVersionWords(server.agentVersion)}{" "}
|
||||
{agentState(server, detail.agentLatest) === "behind" ? (
|
||||
<span className="badge bg-yellow-lt text-yellow" title="Run its install command again to update it (Servers → Manage servers).">
|
||||
<span className="badge bg-yellow-lt text-yellow" title="Run its install command again to update it (Servers → Manage servers), or switch on automatic agent updates on the Servers page.">
|
||||
update available
|
||||
</span>
|
||||
) : (
|
||||
@@ -807,6 +808,7 @@ export default function ServerDetail({ user }: { user: CurrentUser }) {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<HostHealthCard health={server.hostHealth} />
|
||||
{hardware.source === "agent" && <ServerTrends serverId={serverId} />}
|
||||
{(user.role === "admin" || user.role === "operator") && <ServerActivity serverId={serverId} />}
|
||||
<div className="card mb-3">
|
||||
|
||||
@@ -16,6 +16,7 @@ import { tagBadge, useTagColors } from "../utils/tags";
|
||||
import { confirmDialog } from "../utils/dialogs";
|
||||
import { ErrorNotice } from "../components/Notice";
|
||||
import { toast } from "../utils/toast";
|
||||
import { readableError } from "../utils/errors";
|
||||
import { SearchBox } from "../components/TableToolbar";
|
||||
import { useTableSearch } from "../hooks/useTableSearch";
|
||||
import SaveViewButton from "../components/SaveViewButton";
|
||||
@@ -36,6 +37,8 @@ export default function Servers({ user }: { user: CurrentUser }) {
|
||||
// Until the first answer arrives there is nothing to say about "no servers", so show the page's shape instead of the empty message.
|
||||
const [loaded, setLoaded] = useState(false);
|
||||
const [agentLatest, setAgentLatest] = useState<AgentLatest | null>(null);
|
||||
// Whether agents that are behind replace themselves (an administrator's switch).
|
||||
const [autoUpdate, setAutoUpdate] = useState<boolean | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [managingServers, setManagingServers] = useState(false);
|
||||
const [integrationColors, setIntegrationColors] = useState<Record<string, string>>({});
|
||||
@@ -76,6 +79,7 @@ export default function Servers({ user }: { user: CurrentUser }) {
|
||||
.then((res) => {
|
||||
setServers(res.servers);
|
||||
setAgentLatest(res.agentLatest);
|
||||
setAutoUpdate(res.agentAutoUpdate);
|
||||
})
|
||||
.catch((err) => setError(String(err)))
|
||||
.finally(() => setLoaded(true));
|
||||
@@ -85,6 +89,17 @@ export default function Servers({ user }: { user: CurrentUser }) {
|
||||
loadServers();
|
||||
}, [loadServers]);
|
||||
|
||||
async function changeAutoUpdate(value: boolean) {
|
||||
setAutoUpdate(value);
|
||||
try {
|
||||
await api.servers.setAgentAutoUpdate(value);
|
||||
toast.success(value ? "Agents that are behind will update themselves at their next run." : "Agents will no longer update themselves.");
|
||||
} catch (err) {
|
||||
setAutoUpdate(!value);
|
||||
toast.error(readableError(err));
|
||||
}
|
||||
}
|
||||
|
||||
async function createServer(e: React.FormEvent) {
|
||||
e.preventDefault();
|
||||
setError(null);
|
||||
@@ -245,6 +260,21 @@ export default function Servers({ user }: { user: CurrentUser }) {
|
||||
</div>
|
||||
<ErrorNotice error={error} />
|
||||
|
||||
{isAdmin && autoUpdate !== null && servers.length > 0 && (
|
||||
<div className="card mb-3">
|
||||
<div className="card-body py-2">
|
||||
<label className="form-check form-switch mb-0">
|
||||
<input className="form-check-input" type="checkbox" checked={autoUpdate} onChange={(e) => void changeAutoUpdate(e.target.checked)} />
|
||||
<span className="form-check-label">Update agents automatically</span>
|
||||
</label>
|
||||
<div className="text-secondary small mt-1">
|
||||
An agent that is behind this app downloads the newer script from here, checks it against a checksum the server gives, and replaces itself at its next run. Off by default. An agent needs the
|
||||
update code to use it (Linux version 4 or later, Windows 3 or later); an older one needs its install command run once (Manage servers).
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{managingServers ? (
|
||||
<>
|
||||
<div className="card mb-3">
|
||||
|
||||
Reference in new issue
Block a user