Host health from the Linux agent, agent self-update, and disk forecast in /metrics

- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates;
  SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox
- Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new
  "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series
- Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified
  download, atomic replace; GET /api/agent/report/update
- homelab_server_disk_days_to_full in /metrics
- Tests (server, real PowerShell against a fake server) and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 00:54:24 +02:00
1 parent 540ab704b7
commit 5dc6cea69c
38 files changed
+3884 -73

No files matched your search

+180
View File
@@ -0,0 +1,180 @@
/**
* The Windows agent replacing itself: only with a script the server vouched for (SHA-256), that is newer, says the version promised and
* parses; and only when it is installed under its own name. Run against a fake server, with the real script's own function.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { spawn, spawnSync } from "node:child_process";
import http from "node:http";
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
const realScript = join(repoRoot, "agent", "windows", "report-tasks.ps1");
const probe = process.platform === "win32" ? spawnSync("pwsh", ["-NoProfile", "-Command", "$PSVersionTable.PSVersion.Major"], { encoding: "utf8" }) : null;
const havePwsh = probe?.status === 0;
const sha = (b: Buffer | string) => createHash("sha256").update(b).digest("hex");
/** Runs PowerShell without blocking this process: the fake server these tests talk to lives in it, so a blocking spawn would deadlock. */
function pwshFile(file: string): Promise<{ stdout: string; stderr: string }> {
return new Promise((resolve, reject) => {
const child = spawn("pwsh", ["-NoProfile", "-NonInteractive", "-File", file]);
let stdout = "";
let stderr = "";
const timer = setTimeout(() => child.kill(), 60_000);
child.stdout.on("data", (c) => (stdout += c));
child.stderr.on("data", (c) => (stderr += c));
child.on("error", reject);
child.on("close", () => {
clearTimeout(timer);
resolve({ stdout, stderr });
});
});
}
describe("the Windows agent updating itself", { skip: !havePwsh }, () => {
let work = "";
let fake: http.Server;
let port = 0;
// What the fake server answers; each test sets it.
let reply: { update: boolean; version: number; sha256: string } = { update: false, version: 3, sha256: "" };
let served: Buffer = Buffer.alloc(0);
let asked: string[] = [];
before(async () => {
work = mkdtempSync(join(tmpdir(), "hm-winupdate-"));
fake = http.createServer((req, res) => {
asked.push(`${req.method} ${req.url} auth=${req.headers.authorization ?? "-"}`);
if (req.url?.startsWith("/api/agent/report/update")) {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify(reply));
} else if (req.url === "/agent/windows/report-tasks.ps1") {
res.writeHead(200, { "content-type": "text/plain" });
res.end(served);
} else {
res.writeHead(404).end();
}
});
await new Promise<void>((r) => fake.listen(0, "127.0.0.1", r));
port = (fake.address() as { port: number }).port;
});
after(() => {
fake?.close();
try {
rmSync(work, { recursive: true, force: true });
} catch {
// best effort
}
});
const real = () => readFileSync(realScript);
const OLD = "# the installed agent\n$script:AgentVersion = 1\nWrite-Output 'old'\n";
/** Installs an old stub under `name`, runs the real Update-AgentSelf against the fake server, returns what is left. */
async function attempt(name = "homelab-manager-agent.ps1") {
const dir = mkdtempSync(join(work, "run-"));
mkdirSync(dir, { recursive: true });
const installed = join(dir, name);
writeFileSync(installed, OLD);
asked = [];
const script = `
$ErrorActionPreference = 'Stop'
. '${realScript.replace(/'/g, "''")}'
$script:AgentVersion = 1
$config = @{ ApiUrl = 'http://127.0.0.1:${port}'; ApiToken = 'secret-token'; Insecure = $false }
Update-AgentSelf $config -Self '${installed.replace(/'/g, "''")}'
'DONE'
`;
const scriptFile = join(dir, "run.ps1");
writeFileSync(scriptFile, script);
const r = await pwshFile(scriptFile);
assert.ok(r.stdout.includes("DONE"), `the script did not finish:\n${r.stdout}\n${r.stderr}`);
return { content: readFileSync(installed), leftovers: readdirSync(dir).filter((f) => f.endsWith(".download")), stderr: r.stderr, asked: [...asked] };
}
test("a newer script that matches the checksum replaces the old one, and the token is sent only to the question", async () => {
served = real();
reply = { update: true, version: 3, sha256: sha(served) };
const out = await attempt();
assert.deepEqual(out.content, served, "the installed file is now the served script, byte for byte");
assert.deepEqual(out.leftovers, []);
assert.match(out.stderr, /Updated the agent from version 1 to version 3/);
const question = out.asked.find((l) => l.includes("/api/agent/report/update"))!;
assert.match(question, /version=1/);
assert.match(question, /auth=Bearer secret-token/);
const download = out.asked.find((l) => l.includes("/agent/windows/report-tasks.ps1"))!;
assert.match(download, /auth=-$/, "the public script is fetched without the token");
});
test("nothing happens when the server says no", async () => {
served = real();
reply = { update: false, version: 3, sha256: sha(served) };
const out = await attempt();
assert.equal(out.content.toString(), OLD);
assert.equal(out.asked.some((l) => l.includes("/agent/windows/")), false, "and it does not even download");
});
test("a download that doesn't match the checksum is thrown away", async () => {
served = real();
reply = { update: true, version: 3, sha256: sha("something else") };
const out = await attempt();
assert.equal(out.content.toString(), OLD);
assert.deepEqual(out.leftovers, []);
assert.match(out.stderr, /didn't check out/);
});
test("a script that is not newer, doesn't say the promised version, or doesn't parse is refused even if the checksum fits", async () => {
// not newer than the running version
served = real();
reply = { update: true, version: 1, sha256: sha(served) };
assert.equal((await attempt()).content.toString(), OLD, "version 1 is not newer than 1");
// says a different version than was promised
served = Buffer.from("$script:AgentVersion = 2\nWrite-Output 'x'\n");
reply = { update: true, version: 3, sha256: sha(served) };
assert.equal((await attempt()).content.toString(), OLD);
// says the right version but is not a script
served = Buffer.from("$script:AgentVersion = 3\nfunction Broken { if ( \n");
reply = { update: true, version: 3, sha256: sha(served) };
assert.equal((await attempt()).content.toString(), OLD);
});
test("a copy that isn't installed under the agent's own name is never replaced", async () => {
served = real();
reply = { update: true, version: 3, sha256: sha(served) };
const out = await attempt("report-tasks.ps1");
assert.equal(out.content.toString(), OLD);
assert.equal(out.asked.length, 0, "it does not even ask");
});
test("a server that can't be reached costs nothing", async () => {
const dir = mkdtempSync(join(work, "run-"));
const installed = join(dir, "homelab-manager-agent.ps1");
writeFileSync(installed, OLD);
const script = `
$ErrorActionPreference = 'Stop'
. '${realScript.replace(/'/g, "''")}'
$script:AgentVersion = 1
$config = @{ ApiUrl = 'http://127.0.0.1:1'; ApiToken = 'x'; Insecure = $false }
Update-AgentSelf $config -Self '${installed.replace(/'/g, "''")}'
'DONE'
`;
const scriptFile = join(dir, "run.ps1");
writeFileSync(scriptFile, script);
const r = await pwshFile(scriptFile);
assert.ok(r.stdout.includes("DONE"));
assert.match(r.stderr, /couldn't check for, or install, a newer agent/);
assert.equal(readFileSync(installed, "utf8"), OLD);
});
test("the script stays plain ASCII, as its own header asks", () => {
assert.ok(!/[^\x00-\x7f]/.test(readFileSync(realScript, "utf8")));
});
});