Host health from the Linux agent, agent self-update, and disk forecast in /metrics

- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates;
  SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox
- Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new
  "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series
- Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified
  download, atomic replace; GET /api/agent/report/update
- homelab_server_disk_days_to_full in /metrics
- Tests (server, real PowerShell against a fake server) and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 00:54:24 +02:00
1 parent 540ab704b7
commit 5dc6cea69c
38 files changed
+3884 -73

No files matched your search

+3
View File
@@ -297,6 +297,9 @@ export const servers = sqliteTable("servers", {
// When updates were first seen waiting (and none were before) — how long the oldest has been waiting. Null when none are.
updatesPendingSince: text("updates_pending_since"),
// What the agent says about the machine's own health, as JSON: {failedUnits, smart, zfs, reportedAt} — see services/hostHealth.ts. Null until it says.
hostHealth: text("host_health"),
// Which version of the agent script is running on it, as the agent says in each report (see agent/*/report-tasks.*). Null until it says:
// an agent that predates the number being reported is version 1.
agentVersion: integer("agent_version"),
+45
View File
@@ -4,6 +4,8 @@ import { db } from "../db/client.js";
import { hashToken } from "../services/tokens.js";
import { syncServerTasks } from "../services/taskSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { getSettings } from "../services/settingsStore.js";
import { latestAgentSha256, latestAgentVersions } from "../services/agentVersions.js";
export const agentReportRouter = Router();
@@ -31,7 +33,33 @@ const updatesSchema = z.object({
reboot_required: z.boolean().optional(),
});
// Failed services, disk health, ZFS pools. Each part is checked on its own and a bad one dropped, like the rest: it must never cost the report.
const healthSchema = z.object({
failed_units: z
.array(z.object({ unit: text(200), description: text(300).optional() }))
.transform(cap(100))
.optional()
.catch(undefined),
smart: z
.array(z.object({ device: text(100), model: text(200).optional(), healthy: z.boolean().nullable().optional(), temperature_c: z.number().nullable().optional(), wear_percent: z.number().nullable().optional() }))
.transform(cap(100))
.optional()
.catch(undefined),
zfs: z
.array(z.object({ name: text(200), health: text(40), capacity_percent: z.number().nullable().optional() }))
.transform(cap(100))
.optional()
.catch(undefined),
});
const systemSchema = z.object({
health: z
.unknown()
.optional()
.transform((h) => {
const parsed = healthSchema.safeParse(h);
return parsed.success && Object.values(parsed.data).some((v) => v !== undefined) ? parsed.data : undefined;
}),
updates: z
.unknown()
.optional()
@@ -105,6 +133,23 @@ async function authenticateAgent(req: Request, res: Response, next: NextFunction
next();
}
/**
* "Should I replace myself?" — asked by an agent that has been told to keep itself up to date, before it reports. Says yes only when an
* administrator has switched automatic updates on and this agent is behind the script this app serves, and gives the script's SHA-256 so
* the agent can check what it downloads. The agent says which version it is; failing that, the last one it reported counts.
*/
agentReportRouter.get("/update", asyncHandler(authenticateAgent), asyncHandler(async (req, res) => {
const server = res.locals.agentServer as { osType: string; agentVersion: number | null };
const os = server.osType === "windows" ? "windows" : "linux";
const claimed = Number(req.query.version);
const running = Number.isInteger(claimed) && claimed >= 1 && claimed <= 100_000 ? claimed : (server.agentVersion ?? 1);
const latest = latestAgentVersions()[os];
const sha256 = latestAgentSha256(os);
const { agents } = await getSettings();
const update = agents.autoUpdate && latest !== null && sha256 !== null && running < latest;
res.json({ update, version: update ? latest : null, sha256: update ? sha256 : null });
}));
agentReportRouter.post("/", asyncHandler(authenticateAgent), express.json({ limit: AGENT_BODY_LIMIT }), asyncHandler(async (req, res) => {
const server = res.locals.agentServer as { id: number };
+1
View File
@@ -20,6 +20,7 @@ alertTicketsRouter.use(requireAuth, requireRole("admin"));
const CATEGORIES = [
{ value: "offline", label: "Server down" },
{ value: "disk", label: "Disk & storage" },
{ value: "services", label: "Failing services" },
{ value: "backup", label: "Backups" },
{ value: "updates", label: "Updates" },
{ value: "expiry", label: "Expiry" },
+1 -1
View File
@@ -15,7 +15,7 @@ runbooksRouter.use(requireAuth);
/** More than anyone needs for one thing; each is another note to keep true. */
const MAX_PER_TARGET = 20;
export const ALERT_CATEGORIES = ["offline", "disk", "backup", "updates", "expiry", "automation", "integration", "monitoring", "tickets"] as const;
export const ALERT_CATEGORIES = ["offline", "disk", "services", "backup", "updates", "expiry", "automation", "integration", "monitoring", "tickets"] as const;
const bodySchema = z.object({
title: z.string().trim().min(1).max(120),
+19 -5
View File
@@ -8,6 +8,7 @@ import { generateApiToken } from "../services/tokens.js";
import { recordAudit } from "../services/audit.js";
import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js";
import { parsePendingUpdates } from "../services/pendingUpdates.js";
import { parseHostHealth } from "../services/hostHealth.js";
import { latestAgentVersions } from "../services/agentVersions.js";
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
import { getServerActivity } from "../services/serverActivity.js";
@@ -19,14 +20,14 @@ import { serverPortsRouter } from "./serverPorts.js";
import { InvalidTagError, normalizeTags, parseTags } from "../services/serverTags.js";
import { buildServerSummary } from "../services/serverSummary.js";
import { activeSubjects } from "../services/maintenance.js";
import { getSettings } from "../services/settingsStore.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
export const serversRouter = Router();
/** A server row as the API returns it: no token hash, and tags as an array rather than the stored JSON. */
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null }>(row: T) {
const { apiTokenHash: _hash, tags, pendingUpdates, ...rest } = row;
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates) };
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null; hostHealth: string | null }>(row: T) {
const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, ...rest } = row;
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth) };
}
serversRouter.use(requireAuth);
@@ -43,11 +44,23 @@ serversRouter.get("/", asyncHandler(async (_req, res) => {
const rows = await db.query.servers.findMany({ orderBy: (s, { asc }) => [asc(s.name)] });
res.json({
servers: rows.map(publicServer),
// Whether agents that are behind replace themselves, so the page can show the switch.
agentAutoUpdate: (await getSettings()).agents.autoUpdate,
// What a new install would be, so the page can say which agents are behind.
agentLatest: latestAgentVersions(),
});
}));
// Whether agents that are behind download the newer script from here and replace themselves. An administrator's to switch.
serversRouter.put("/agent-update", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = z.object({ autoUpdate: z.boolean() }).safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body" });
const before = (await getSettings()).agents.autoUpdate;
await updateSettings({ agents: { autoUpdate: parsed.data.autoUpdate } });
await recordAudit({ actor: req.currentUser!, category: "settings", action: "agent_auto_update", targetType: "settings", detail: { from: before, to: parsed.data.autoUpdate } });
res.json({ autoUpdate: parsed.data.autoUpdate });
}));
serversRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = createServerSchema.safeParse(req.body);
if (!parsed.success) {
@@ -353,6 +366,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
listeningPorts: _listeningPorts,
lastPortScan: _lastPortScan,
pendingUpdates: rawUpdates,
hostHealth: rawHealth,
...serverOut
} = server;
@@ -366,7 +380,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
children: [...links2.entries()].filter(([, l]) => l.parentId === id).map(([childId, l]) => ({ id: childId, name: nameOf.get(childId) ?? "", auto: l.auto })).sort((a, b) => a.name.localeCompare(b.name)),
};
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates) }, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: parseHostHealth(rawHealth) }, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
}));
const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) });
+16 -1
View File
@@ -6,8 +6,11 @@
*
* 1 — before the number was reported (so an agent that says nothing is version 1)
* 2 — reports pending updates, and its own version
* 3 — Linux: the package manager is asked outside the systemd sandbox, so dnf and zypper can answer too (Windows stays at 2)
* 3 — Linux: the package manager is asked outside the systemd sandbox, so dnf and zypper can answer too.
* Windows: can replace itself when an administrator has switched automatic updates on.
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too
*/
import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
@@ -51,6 +54,18 @@ export function latestAgentVersions(dir: string = AGENT_DIR): LatestAgentVersion
return result;
}
/**
* The SHA-256 of the script this app serves for an OS, as hex — what an agent that updates itself checks the download against, so a
* truncated or tampered-with transfer is never installed. Null if the script can't be read.
*/
export function latestAgentSha256(os: AgentOs, dir: string = AGENT_DIR): string | null {
try {
return createHash("sha256").update(readFileSync(join(dir, FILES[os]))).digest("hex");
} catch {
return null;
}
}
/**
* Whether a server's agent is behind. Only a server that reports through an agent can be: one that never has (a Proxmox guest tracked
* through its host, say) has none to update. An agent that reports no version is version 1.
+1
View File
@@ -126,6 +126,7 @@ export function planTicketRun(input: {
const CATEGORY_LABELS: Record<AlertCategory, string> = {
offline: "Server down",
disk: "Disk & storage",
services: "Failing services",
backup: "Backups",
updates: "Updates",
expiry: "Expiry",
+1 -1
View File
@@ -4,7 +4,7 @@
export type AlertSeverity = "critical" | "warning" | "info";
/** What kind of problem — drives the filter on the Alerts page. */
export type AlertCategory = "offline" | "disk" | "backup" | "updates" | "expiry" | "automation" | "integration" | "monitoring" | "tickets";
export type AlertCategory = "offline" | "disk" | "services" | "backup" | "updates" | "expiry" | "automation" | "integration" | "monitoring" | "tickets";
export interface Alert {
/** Stable, so the page can key rows on it. */
+2 -2
View File
@@ -68,8 +68,8 @@ const SEVERITY_ORDER: Record<AlertSeverity, number> = { critical: 0, warning: 1,
/** Which kind of source, and which one, a health/automation condition key belongs to. */
function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } {
let m = /^(?:offline|disk):server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : "disk" };
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : "disk" };
m = /^disk:proxmox:(\d+):/.exec(key);
if (m) return { type: "proxmox", id: Number(m[1]), category: "disk" };
m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key);
+5 -1
View File
@@ -9,6 +9,7 @@ import { recordAudit } from "./audit.js";
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
import { hostHealthConditions, parseHostHealth, type HostHealth } from "./hostHealth.js";
const STATE_FLAG = "healthActiveConditions";
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
@@ -32,6 +33,8 @@ export interface ServerSnapshot {
name: string;
lastSeenAt: string | null;
disks: { mount: string; sizeBytes: number; usedBytes: number }[];
/** What the agent says about failed services, disk health and ZFS pools, if it says anything. */
health?: HostHealth | null;
}
export interface ProxmoxSnapshot {
integrationId: number;
@@ -115,6 +118,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
}
// A server that never reported has no agent to go quiet; and an offline one's disk figures are stale, so neither is judged on disks.
if (s.lastSeenAt && !offline) {
for (const c of hostHealthConditions(s.id, s.name, s.health ?? null)) out.push({ key: c.key, source: "server", message: c.message, severity: c.severity });
for (const d of s.disks) {
const p = usage(d.usedBytes, d.sizeBytes);
if (p !== null && p >= limit) {
@@ -296,7 +300,7 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
} catch {
// a malformed report shouldn't stop every other server being checked
}
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks });
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: parseHostHealth(s.hostHealth) });
}
const rows = await db
+124
View File
@@ -0,0 +1,124 @@
/**
* What a Linux agent can say about the machine's own health, beyond its figures: services that have failed (systemd), whether each disk's
* SMART self-assessment passes and how worn it is, and whether each ZFS pool is online. Each part is optional — an agent reports only what
* it could find out (no smartctl, no ZFS, no systemd) — and a part that is absent from a report keeps what was last said, so an agent that
* skips one never turns an old problem into "all well".
*/
export interface FailedUnit {
unit: string;
description: string;
}
export interface SmartDisk {
device: string;
model: string;
/** The drive's own overall verdict: true passes, false fails, null if it didn't give one. */
healthy: boolean | null;
temperatureC: number | null;
/** NVMe and some SSDs say how much of their rated life is used, 0–100 (and past it). */
wearPercent: number | null;
}
export interface ZfsPool {
name: string;
/** ONLINE, DEGRADED, FAULTED, UNAVAIL, SUSPENDED … as zpool says it. */
health: string;
capacityPercent: number | null;
}
export interface HostHealth {
failedUnits?: FailedUnit[];
smart?: SmartDisk[];
zfs?: ZfsPool[];
/** ISO — when the agent last told us anything here. */
reportedAt: string;
}
/** As it arrives from an agent. */
export interface IncomingHostHealth {
failed_units?: { unit: string; description?: string }[];
smart?: { device: string; model?: string; healthy?: boolean | null; temperature_c?: number | null; wear_percent?: number | null }[];
zfs?: { name: string; health: string; capacity_percent?: number | null }[];
}
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
export function parseHostHealth(stored: string | null): HostHealth | null {
if (!stored) return null;
try {
const v = JSON.parse(stored);
if (!v || typeof v !== "object") return null;
const out: HostHealth = { reportedAt: typeof v.reportedAt === "string" ? v.reportedAt : "" };
if (Array.isArray(v.failedUnits)) out.failedUnits = v.failedUnits.filter((u: any) => u && typeof u.unit === "string").map((u: any) => ({ unit: u.unit, description: typeof u.description === "string" ? u.description : "" }));
if (Array.isArray(v.smart)) {
out.smart = v.smart
.filter((d: any) => d && typeof d.device === "string")
.map((d: any) => ({ device: d.device, model: typeof d.model === "string" ? d.model : "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperatureC), wearPercent: num(d.wearPercent) }));
}
if (Array.isArray(v.zfs)) out.zfs = v.zfs.filter((p: any) => p && typeof p.name === "string" && typeof p.health === "string").map((p: any) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacityPercent) }));
return out;
} catch {
return null;
}
}
/** The text to store after a report: the parts the agent sent replace what was there, the parts it didn't send stay. */
export function nextHostHealth(previous: string | null, incoming: IncomingHostHealth, now: string): string | null {
const keep = parseHostHealth(previous);
const next: HostHealth = { ...(keep ?? {}), reportedAt: now };
let any = false;
if (incoming.failed_units) {
any = true;
next.failedUnits = incoming.failed_units.map((u) => ({ unit: u.unit, description: u.description ?? "" }));
}
if (incoming.smart) {
any = true;
next.smart = incoming.smart.map((d) => ({ device: d.device, model: d.model ?? "", healthy: typeof d.healthy === "boolean" ? d.healthy : null, temperatureC: num(d.temperature_c), wearPercent: num(d.wear_percent) }));
}
if (incoming.zfs) {
any = true;
next.zfs = incoming.zfs.map((p) => ({ name: p.name, health: p.health, capacityPercent: num(p.capacity_percent) }));
}
return any ? JSON.stringify(next) : previous;
}
/** A pool in any state but ONLINE has lost something (DEGRADED has lost redundancy, the rest are worse). */
export const poolIsFine = (p: ZfsPool) => p.health.toUpperCase() === "ONLINE";
/** Wear at which a drive is worth replacing soon, in percent of rated life used. */
export const WEAR_WARNING_PERCENT = 90;
export interface HostHealthCondition {
key: string;
message: string;
severity: "critical" | "warning";
}
const listOf = (names: string[], max = 4) => (names.length <= max ? names.join(", ") : `${names.slice(0, max).join(", ")} and ${names.length - max} more`);
/** The problems a server's health report amounts to. Pure, so the rules can be tested exactly. */
export function hostHealthConditions(serverId: number, serverName: string, health: HostHealth | null): HostHealthCondition[] {
if (!health) return [];
const out: HostHealthCondition[] = [];
const failed = health.failedUnits ?? [];
if (failed.length > 0) {
out.push({
key: `failed-units:server:${serverId}`,
severity: "warning",
message: `${serverName}: ${failed.length === 1 ? "1 service has" : `${failed.length} services have`} failed — ${listOf(failed.map((u) => u.unit))}`,
});
}
for (const d of health.smart ?? []) {
const label = `${d.device}${d.model ? ` (${d.model})` : ""}`;
if (d.healthy === false) {
out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "critical", message: `${serverName}: the disk ${label} says it is failing (SMART self-assessment)` });
} else if (d.wearPercent !== null && d.wearPercent >= WEAR_WARNING_PERCENT) {
out.push({ key: `smart:server:${serverId}:${d.device}`, severity: "warning", message: `${serverName}: the disk ${label} has used ${Math.round(d.wearPercent)}% of its rated life` });
}
}
for (const p of health.zfs ?? []) {
if (!poolIsFine(p)) {
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: ZFS pool ${p.name} is ${p.health}` });
}
}
return out;
}
+1 -1
View File
@@ -23,7 +23,7 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
* so conditions persisted by an earlier version still map correctly.
*/
export function subjectOfConditionKey(key: string): string | null {
let m = /^(?:offline|disk):server:(\d+)(?::|$)/.exec(key);
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)(?::|$)/.exec(key);
if (m) return `server:${m[1]}`;
m = /^disk:proxmox:(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
+20
View File
@@ -10,6 +10,8 @@ import { getAlerts } from "./alerts.js";
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
import { getSourceStatus } from "./integrationHealthMonitor.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { computeSecretStatus } from "./secretStatus.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
@@ -80,8 +82,26 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
if (updates.security !== null) out.add("homelab_server_pending_security_updates", "gauge", "Of those, the ones that are security fixes.", updates.security, { server });
out.add("homelab_server_reboot_required", "gauge", "1 if a restart is due.", updates.rebootRequired ? 1 : 0, { server });
}
const health = parseHostHealth(s.hostHealth);
if (health?.failedUnits) out.add("homelab_server_failed_units", "gauge", "Services (systemd units) that have failed on the server.", health.failedUnits.length, { server });
for (const d of health?.smart ?? []) {
if (d.healthy !== null) out.add("homelab_server_disk_healthy", "gauge", "1 if the disk's own SMART self-assessment passes, 0 if it fails.", d.healthy ? 1 : 0, { server, device: d.device });
if (d.wearPercent !== null) out.add("homelab_server_disk_wear_percent", "gauge", "How much of an SSD's rated life is used.", d.wearPercent, { server, device: d.device });
if (d.temperatureC !== null) out.add("homelab_server_disk_temperature_celsius", "gauge", "A disk's temperature.", d.temperatureC, { server, device: d.device });
}
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a ZFS pool is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
out.add("homelab_server_agent_outdated", "gauge", "1 if the server's agent is older than the one this installation would install now.", isAgentOutdated(s, latest) ? 1 : 0, { server });
}
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
try {
for (const s of serverRows) {
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), 3650, 101)) {
out.add("homelab_server_disk_days_to_full", "gauge", "Days until a disk is full at its present rate of filling; only for disks with a steady climb.", f.daysToFull, { server: s.name, mount: f.mount });
}
}
} catch (err) {
console.error("[metrics] couldn't work out the disk forecasts:", err);
}
try {
const uptime = await getUptime(now, 30);
for (const s of serverRows) {
+2
View File
@@ -54,6 +54,8 @@ export function describeAudit(category: string, action: string, detail: Record<s
return { title: "A disk got nearly full", tone: "bad", kind: "health", detail: message };
case "disk_ok":
return { title: "A disk is no longer nearly full", tone: "good", kind: "health", detail: null };
case "agent_updated":
return { title: `Agent updated to version ${detail.to ?? "?"}`, tone: "good", kind: "change", detail: detail.from !== undefined ? `from version ${detail.from}` : null };
case "create":
return { title: "Server added", tone: "neutral", kind: "change", detail: null };
case "update":
+8
View File
@@ -119,6 +119,12 @@ export interface SignInSettings {
otherwise: "viewer" | "keep";
}
/** How the agents out on the servers are kept up to date. Off until an administrator turns it on. */
export interface AgentSettings {
/** An agent that is behind downloads the newer script from this app, checks it and replaces itself. */
autoUpdate: boolean;
}
/** What the inventory needs to work out a running cost. Without a price per kWh it shows draw but no cost. */
export interface InventorySettings {
/** A short name shown after amounts ("SEK", "kr", "$"); may be blank. */
@@ -180,6 +186,7 @@ export interface AppSettings {
digest: DigestSettings;
signIn: SignInSettings;
inventory: InventorySettings;
agents: AgentSettings;
consistency: ConsistencySettings;
nameGenerator: NameGeneratorSettings;
alertTickets: AlertTicketSettings;
@@ -214,6 +221,7 @@ const DEFAULTS: AppSettings = {
quietHours: { enabled: false, start: "22:00", end: "07:00" },
digest: { enabled: false, day: 1, time: "08:00" },
inventory: { currency: "", pricePerKwh: null },
agents: { autoUpdate: false },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
+15 -1
View File
@@ -3,6 +3,8 @@ import { db } from "../db/client.js";
import { scheduledTasks, servers } from "../db/schema.js";
import { recordServerMetrics } from "./serverMetrics.js";
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
import { nextHostHealth, type IncomingHostHealth } from "./hostHealth.js";
import { recordAudit } from "./audit.js";
export interface IncomingTask {
scheduleType: "cron" | "systemd_timer" | "windows_task";
@@ -23,6 +25,8 @@ export interface IncomingSystemInfo {
listening_ports?: { protocol: "tcp" | "udp"; port: number; address: string; process?: string }[];
/** Updates waiting to be installed. Absent from an agent that doesn't report them (or couldn't find out this time). */
updates?: IncomingUpdates;
/** Failed services, disk health, ZFS pools — each part present only if the agent could find it out. */
health?: IncomingHostHealth;
}
export interface AgentReport {
@@ -103,13 +107,14 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
}
const system = report.system;
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince }).from(servers).where(eq(servers.id, serverId));
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, hostHealth: servers.hostHealth, agentVersion: servers.agentVersion }).from(servers).where(eq(servers.id, serverId));
await db
.update(servers)
.set({
lastSeenAt: now,
// Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before.
...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}),
...(system?.health && before ? { hostHealth: nextHostHealth(before.hostHealth, system.health, now) } : {}),
...(report.hostname ? { hostname: report.hostname } : {}),
...(report.agentVersion !== undefined ? { agentVersion: report.agentVersion } : {}),
// An agent knows what it runs on better than whoever registered the server did.
@@ -132,6 +137,15 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
})
.where(eq(servers.id, serverId));
// An agent that now says a higher version than it did has been updated (by hand, or by itself): worth a line in the server's history.
if (before && report.agentVersion !== undefined && before.agentVersion !== null && report.agentVersion > before.agentVersion) {
try {
await recordAudit({ category: "server", action: "agent_updated", targetType: "server", targetId: serverId, detail: { from: before.agentVersion, to: report.agentVersion } });
} catch (err) {
console.error("[agent] couldn't record the agent update:", err);
}
}
// Keep a little history of the figures too. A failure here must not lose the report itself, which is already saved.
try {
await recordServerMetrics(serverId, system, new Date(now));