Host health from the Linux agent, agent self-update, and disk forecast in /metrics

- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates;
  SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox
- Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new
  "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series
- Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified
  download, atomic replace; GET /api/agent/report/update
- homelab_server_disk_days_to_full in /metrics
- Tests (server, real PowerShell against a fake server) and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 00:54:24 +02:00
1 parent 540ab704b7
commit 5dc6cea69c
38 files changed
+3884 -73

No files matched your search

+26 -3
View File
@@ -361,7 +361,7 @@ describe("Linux agent", () => {
// The one step that runs outside the sandbox (+) is the package-manager question, which may fail without stopping the report (-),
// and it hands its answer over through the unit's own private directory, which is where the script looks for it.
assert.equal(settings.get("ExecStartPre"), "-+/usr/local/bin/homelab-manager-agent.sh --prepare-updates");
assert.equal(settings.get("ExecStartPre"), "-+/usr/local/bin/homelab-manager-agent.sh --prepare");
assert.equal(rendered.split("\n").filter((l) => /^ExecStartPre=/.test(l.trim())).length, 1, "only that one step is outside the sandbox");
assert.equal(settings.get("RuntimeDirectory"), "homelab-manager-agent");
assert.equal(settings.get("RuntimeDirectoryMode"), "0700");
@@ -399,16 +399,39 @@ describe("Linux agent", () => {
test("the update check can be run on its own, and what it leaves behind is read back through jq, not trusted", () => {
const text = readFileSync(agentScript, "utf8");
assert.ok(text.includes("--prepare-updates) MODE=prepare"));
assert.ok(text.includes("--check-updates) MODE=check"));
assert.ok(text.includes("--prepare|--prepare-updates) MODE=prepare"), "the old name still works, for units written by an earlier installer");
assert.ok(text.includes("--check-updates|--check) MODE=check"));
assert.ok(text.includes("use_prepared_updates || collect_updates"), "the report uses the prepared answer first and asks itself otherwise");
assert.ok(text.includes('find "$UPDATES_FILE" -mmin -10'), "an old answer isn't used");
assert.ok(text.includes("jq -ce 'select(type == \"object\""), "the file is re-shaped by jq, so the report only carries the four known fields");
assert.ok(text.includes('mv -f "$UPDATES_FILE.tmp" "$UPDATES_FILE"'), "written whole, then moved into place");
// disk and pool health need the devices, so they are read from the prepare step and handed over, re-shaped by jq like the updates
assert.ok(text.includes("use_prepared_health"));
assert.ok(text.includes("smartctl -j -H -A -n standby"), "a sleeping disk is not woken to be asked");
assert.ok(text.includes("zpool list -H -o name,health,capacity"));
assert.ok(text.includes("systemctl list-units --state=failed"));
assert.ok(text.includes("pacman -Qu"));
// the two modes must not need a server address or a token
assert.ok(text.includes('[[ "$MODE" == "report" && ( -z "$API_URL" || -z "$API_TOKEN" ) ]]'));
});
test("self-update takes nothing on trust but a checksum, never runs inside the sandbox, and can not loop", () => {
const text = readFileSync(agentScript, "utf8");
const body = text.slice(text.indexOf("self_update() {"), text.indexOf("# The answer --prepare left for this report"));
assert.ok(body.length > 500, "self_update wasn't found");
assert.ok(body.includes("api_get \"/api/agent/report/update?version=$AGENT_VERSION\""), "it asks the server, with its token, whether it should");
assert.ok(body.includes("sha256sum") && body.includes("\"$got\" != \"$sha\""), "the download is checked against the checksum the server gave");
assert.ok(body.includes("bash -n \"$work/new.sh\"") && body.includes("#!/usr/bin/env bash") && body.includes("^AGENT_VERSION=$version"), "and must look like this script, parse, and say the version promised");
assert.ok(body.includes("(( version <= AGENT_VERSION ))"), "only ever to a newer version");
assert.ok(body.includes("mv -f \"$self.new\" \"$self\""), "put beside the old one and moved over it in one step");
assert.ok(body.includes("HLM_JUST_UPDATED"), "the new version doesn't update again straight away");
// it is only called from the prepare step (outside the sandbox) and from the manual check, never from the report
const calls = text.split("\n").filter((l) => /(^|\s|&& )self_update(\s|$)/.test(l) && !l.includes("self_update()") && !l.trim().startsWith("#"));
assert.equal(calls.length, 2, calls.join(" | "));
const reportBegins = text.indexOf("collect_cron\ncollect_systemd_timers");
assert.ok(reportBegins > 0 && text.indexOf("self_update", reportBegins) === -1, "not after the point where a report begins");
});
test("unit files are made readable by everyone (the umask above them is for the token file)", () => {
const text = readFileSync(installScript, "utf8").replace(/\r\n/g, "\n");
assert.match(text, /chmod 644 "\$SYSTEMD_DIR\/homelab-manager-agent\.service" "\$SYSTEMD_DIR\/homelab-manager-agent\.timer"/);