Files
Homelab-manager/server/test/containerHealth.test.ts
T
bobbanandClaude Sonnet 5.5 9ca83994fb Name ports after the Docker containers that publish them
The health pass keeps the host ports Dockhand says running containers
publish (new table server_container_ports, migration 0080), matched to a
server by its Docker host's name. They name the port on the server's Ports
card and the network Ports table, ahead of a port preset and behind a
label somebody wrote, and in the new-port alert ("8096/tcp jellyfin
(container)"). Never written onto port notes, so a container's port is
still a new port; an unreadable host keeps its names, a day-old name is
dropped. Dockhand is now read on the health pass even with container
alerts off.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-31 22:45:50 +01:00

286 lines
19 KiB
TypeScript

/**
* Containers that are broken: what Docker's status text says, which containers are left alone, when a stop is news and when it is not, what is remembered between looks, how a
* Docker host that cannot be reached is told apart from one with nothing on it, and what the alerts, the Docker page and the "leave alone" button do with it.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-containers-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const c = await import("../src/services/containerHealth.js");
const { evaluateHealth, collectSnapshot, runHealthCheck } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { integrationsRouter } = await import("../src/routes/integrations.js");
const { getSettings, updateSettings } = await import("../src/services/settingsStore.js");
const { auditLog } = await import("../src/db/schema.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const NOW = Date.parse("2026-10-24T10:00:00Z");
const HOUR = 3_600_000;
const info = (name: string, state: string, status: string, env = "Jackie", environmentId = 1): import("../src/services/containerHealth.js").ContainerInfo => ({ environmentId, environmentName: env, name, state, status });
describe("what Docker's status text says", () => {
test("unhealthy, an exit code, and how long ago", () => {
assert.deepEqual(c.parseStatus("Up 6 days (unhealthy)"), { unhealthy: true, exitCode: null, agoMs: null });
assert.deepEqual(c.parseStatus("Up 6 days (healthy)"), { unhealthy: false, exitCode: null, agoMs: null });
assert.deepEqual(c.parseStatus("Restarting (127) 39 seconds ago"), { unhealthy: false, exitCode: 127, agoMs: 39_000 });
assert.deepEqual(c.parseStatus("Exited (0) 15 minutes ago"), { unhealthy: false, exitCode: 0, agoMs: 15 * 60_000 });
assert.equal(c.parseStatus("Exited (2) 3 months ago").agoMs, 3 * 2_592_000_000);
assert.equal(c.parseStatus("Exited (0) About an hour ago").agoMs, HOUR);
assert.equal(c.parseStatus("Exited (0) Less than a second ago").agoMs, 1000);
assert.equal(c.parseStatus("Exited (137) 2 weeks ago").exitCode, 137);
assert.deepEqual(c.parseStatus("Created"), { unhealthy: false, exitCode: null, agoMs: null });
});
test("names to leave alone: exact, with a host, with a *, and the scanners Dockhand starts", () => {
const sub = info("planka-planka-1", "exited", "Exited (0) 2 weeks ago", "Siri");
assert.equal(c.isLeftAlone(sub, []), false);
assert.equal(c.isLeftAlone(sub, ["planka-planka-1"]), true);
assert.equal(c.isLeftAlone(sub, ["Siri/planka-planka-1"]), true);
assert.equal(c.isLeftAlone(sub, ["Jackie/planka-planka-1"]), false, "another host's");
assert.equal(c.isLeftAlone(sub, ["planka-*"]), true);
assert.equal(c.isLeftAlone(sub, ["PLANKA-*"]), true, "capitals don't matter");
assert.equal(c.isLeftAlone(sub, ["planka.planka"]), false, "a dot is a dot");
assert.equal(c.isLeftAlone(info("dockhand-grype-1792611378733-7036sl", "created", "Created"), []), true);
assert.equal(c.isLeftAlone(info("dockhand-trivy-1792847558773-pu9ivk", "running", "Up 25 seconds"), []), true);
});
});
describe("one container at a time", () => {
const judge = (x: ReturnType<typeof info>, prev?: import("../src/services/containerHealth.js").ContainerMemory[string]) => c.judgeContainer(x, prev, NOW);
test("a restart loop is critical, and says the exit code", () => {
const j = judge(info("netbox-netbox-worker-1", "restarting", "Restarting (127) 39 seconds ago"));
assert.deepEqual([j.problem?.kind, j.problem?.severity, j.problem?.text], ["restarting", "critical", "keeps restarting (exit code 127)"]);
});
test("an unhealthy container is a warning; a healthy or plain running one is nothing", () => {
assert.equal(judge(info("drawio", "running", "Up 4 days (unhealthy)")).problem?.kind, "unhealthy");
assert.equal(judge(info("homebox", "running", "Up 5 days (healthy)")).problem, null);
assert.equal(judge(info("ollama", "running", "Up 2 days")).problem, null);
});
test("a stop is news when it was seen happen, or happened within the day; not when it has been so all along", () => {
const iso = new Date(NOW - HOUR).toISOString();
assert.equal(judge(info("grocy", "exited", "Exited (0) 15 minutes ago")).problem?.kind, "stopped", "never seen, stopped a quarter of an hour ago");
assert.equal(judge(info("planka", "exited", "Exited (0) 2 weeks ago")).problem, null, "never seen, down for weeks");
assert.equal(judge(info("vikunja", "exited", "Exited (2) 3 months ago")).problem, null);
assert.equal(judge(info("x", "exited", "Exited (0) 2 weeks ago"), { s: "up", since: iso }).problem?.kind, "stopped", "it was running at the last look");
assert.equal(judge(info("x", "exited", "Exited (0) 2 weeks ago"), { s: "down", since: iso }).problem, null, "it was already down");
assert.equal(judge(info("x", "exited", "Exited (0) 2 days ago"), { s: "down", since: iso, alert: true }).problem?.kind, "stopped", "once raised it stays");
assert.equal(judge(info("x", "exited", "Exited (2) 20 hours ago")).problem?.text, "has stopped (exit code 2)");
assert.equal(judge(info("x", "dead", "Dead"), { s: "up", since: iso }).problem?.text, "is dead");
});
test("created and never started is said only after it has been so for an hour", () => {
const first = judge(info("kestra", "created", "Created"));
assert.equal(first.problem, null);
assert.deepEqual(first.next, { s: "created", since: new Date(NOW).toISOString() });
assert.equal(judge(info("kestra", "created", "Created"), { s: "created", since: new Date(NOW - 30 * 60_000).toISOString() }).problem, null);
assert.equal(judge(info("kestra", "created", "Created"), { s: "created", since: new Date(NOW - 2 * HOUR).toISOString() }).problem?.kind, "created");
});
test("what is remembered: running is up, and a recovered one is no longer alerting", () => {
assert.deepEqual(judge(info("x", "running", "Up 1 hour"), { s: "down", since: "2026-10-23T00:00:00.000Z", alert: true }).next, { s: "up", since: new Date(NOW).toISOString() });
assert.equal(judge(info("x", "paused", "Up 1 hour (Paused)")).problem, null);
});
});
describe("a whole Docker integration, over several looks", () => {
/** A few of the containers from a real export. */
const export1 = [
info("netbox-netbox-worker-1", "restarting", "Restarting (127) 39 seconds ago"),
info("portall-socket-proxy", "running", "Up 6 days (unhealthy)"),
info("drawio", "running", "Up 4 days (unhealthy)", "Siri", 2),
info("grocy", "exited", "Exited (0) 15 minutes ago", "Gabriella", 3),
info("planka-planka-1", "exited", "Exited (0) 2 weeks ago", "Siri", 2),
info("vikunja-vikunja-1", "exited", "Exited (2) 3 months ago", "Nova", 4),
info("nextcloud-aio-watchtower", "exited", "Exited (0) 2 days ago"),
info("snipe-itnew-app-1", "created", "Created", "Nova", 4),
info("dockhand-grype-1792611378733-7036sl", "created", "Created", "Anna", 5),
info("gitea", "running", "Up 4 days"),
];
const snap = (containers = export1, unreachable: import("../src/services/containerHealth.js").DockerSnapshot["unreachable"] = []): import("../src/services/containerHealth.js").DockerSnapshot => ({ integrationId: 7, integrationName: "Dockhand", containers, unreachable });
test("the first look finds what is wrong now, and nothing about what has long been stopped", () => {
const { conditions, next } = c.evaluateContainers([snap()], {}, NOW, []);
assert.deepEqual(conditions.map((x) => x.key).sort(), [
"container-restarting:7:1:netbox-netbox-worker-1",
"container-stopped:7:3:grocy",
"container-unhealthy:7:1:portall-socket-proxy",
"container-unhealthy:7:2:drawio",
]);
const worker = conditions.find((x) => x.key.startsWith("container-restarting"))!;
assert.deepEqual([worker.severity, worker.source, worker.message], ["critical", "dockhand:7:1", "netbox-netbox-worker-1 [Jackie] keeps restarting (exit code 127)"]);
assert.equal(conditions.find((x) => x.key.includes("grocy"))!.message, "grocy [Gabriella] has stopped");
assert.ok(next["7:4:vikunja-vikunja-1"] && !next["7:4:vikunja-vikunja-1"].alert, "remembered as down, quietly");
assert.equal(next["7:5:dockhand-grype-1792611378733-7036sl"], undefined, "the scanner is not remembered at all");
});
test("a stop that was raised stays raised, a container that starts again clears, and one that is left alone is not said", () => {
const first = c.evaluateContainers([snap()], {}, NOW, []);
const later = c.evaluateContainers([snap()], first.next, NOW + 2 * 24 * HOUR, []);
assert.ok(later.conditions.some((x) => x.key.includes("grocy")), "two days on, grocy is still stopped and still said");
assert.ok(!later.conditions.some((x) => x.key.includes("planka")), "planka never was");
const grocyBack = export1.map((x) => (x.name === "grocy" ? info("grocy", "running", "Up 3 minutes", "Gabriella", 3) : x));
const cleared = c.evaluateContainers([snap(grocyBack)], later.next, NOW + 3 * 24 * HOUR, []);
assert.ok(!cleared.conditions.some((x) => x.key.includes("grocy")));
const stoppedAgain = export1.map((x) => (x.name === "gitea" ? info("gitea", "exited", "Exited (1) 5 minutes ago") : x));
const caught = c.evaluateContainers([snap(stoppedAgain)], first.next, NOW + 15 * 60_000, []);
assert.equal(caught.conditions.find((x) => x.key.includes("gitea"))?.message, "gitea [Jackie] has stopped (exit code 1)", "running at the last look, stopped now");
const quiet = c.evaluateContainers([snap()], {}, NOW, ["Siri/drawio", "portall-*", "netbox-netbox-worker-1", "grocy"]);
assert.deepEqual(quiet.conditions, []);
});
test("a Docker host that cannot be read is said, and its containers are neither judged nor forgotten", () => {
const first = c.evaluateContainers([snap()], {}, NOW, []);
const down = c.evaluateContainers([snap(export1.filter((x) => x.environmentId !== 1), [{ environmentId: 1, environmentName: "Jackie", error: "connect ECONNREFUSED" }])], first.next, NOW + HOUR, []);
assert.ok(down.conditions.some((x) => x.key === "container-host:7:1" && x.message === "Dockhand: can't reach the Docker host Jackie — connect ECONNREFUSED"));
assert.ok(!down.conditions.some((x) => x.key === "container-restarting:7:1:netbox-netbox-worker-1"), "not judged while the host is out of reach (the alert is held by the caller)");
assert.ok(down.next["7:1:gitea"], "and what was remembered of Jackie's containers is kept");
const gone = c.evaluateContainers([snap(export1.filter((x) => x.name !== "gitea"))], first.next, NOW + HOUR, []);
assert.equal(gone.next["7:1:gitea"], undefined, "a container that has been removed is forgotten when its host is read");
});
});
describe("through the health pass, the Alerts page and the Docker page", () => {
let dockhand: Server;
let app: Server;
let base = "";
let who = "admin";
let integrationId = 0;
let annaUp = true;
let containers: Record<string, unknown>[] = [];
const row = (name: string, state: string, status: string) => ({ id: `id-${name}`, name, image: "img", state, status });
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
dockhand = createHttpServer((req, res) => {
const url = new URL(req.url ?? "", "http://x");
res.setHeader("content-type", "application/json");
if (url.pathname === "/api/environments") return void res.end(JSON.stringify([{ id: 1, name: "Jackie", connectionType: "hawser" }, { id: 2, name: "Anna", connectionType: "hawser" }]));
if (url.pathname === "/api/containers/pending-updates") return void res.end(JSON.stringify({ pendingUpdates: [] }));
if (url.pathname === "/api/containers") {
if (url.searchParams.get("env") === "2" && !annaUp) {
res.statusCode = 502;
return void res.end(JSON.stringify({ error: "agent not connected" }));
}
return void res.end(JSON.stringify(url.searchParams.get("env") === "1" ? containers : [row("ollama", "running", "Up 2 days")]));
}
res.statusCode = 404;
res.end("{}");
});
await new Promise<void>((r) => dockhand.listen(0, "127.0.0.1", r));
integrationId = (await db.insert(integrations).values({ type: "dockhand", name: "Dockhand", baseUrl: "", config: JSON.stringify({ url: `http://127.0.0.1:${(dockhand.address() as AddressInfo).port}`, token: "t" }) }).returning({ id: integrations.id }))[0].id;
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/integrations", integrationsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/integrations/${integrationId}/dockhand`;
});
after(() => {
dockhand.close();
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass reads Dockhand and finds the broken ones; the Alerts page says them under Docker, with a link", async () => {
containers = [row("netbox-worker", "restarting", "Restarting (127) 39 seconds ago"), row("gitea", "running", "Up 4 days"), row("drawio", "running", "Up 4 days (unhealthy)")];
const { snapshot } = await collectSnapshot();
assert.deepEqual(snapshot.docker?.map((d) => [d.integrationName, d.containers.length, d.unreachable.length]), [["Dockhand", 4, 0]]);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
assert.deepEqual(list.filter((x) => x.key.startsWith("container-")).map((x) => x.key).sort(), [`container-restarting:${integrationId}:1:netbox-worker`, `container-unhealthy:${integrationId}:1:drawio`]);
assert.equal(subjectOfConditionKey(`container-restarting:${integrationId}:1:netbox-worker`), `integration:${integrationId}`, "maintenance on the integration holds it");
invalidateAlertsCache();
const report = await getAlerts(true);
const a = report.alerts.find((x) => x.id === `services:container-restarting:${integrationId}:1:netbox-worker`)!;
assert.ok(a);
assert.deepEqual([a.severity, a.category, a.source, a.link], ["critical", "services", "Docker", "/docker"]);
});
test("a pass remembers what it saw, so a container that stops afterwards is caught; and an unreachable host is said and held", async () => {
await runHealthCheck(Date.now());
const memory = await c.loadContainerMemory();
assert.equal(memory[`${integrationId}:1:gitea`]?.s, "up");
containers = containers.map((x) => (x.name === "gitea" ? row("gitea", "exited", "Exited (1) 2 minutes ago") : x));
annaUp = false;
invalidateAlertsCache();
const report = await getAlerts(true);
const ids = report.alerts.map((x) => x.id);
assert.ok(ids.includes(`services:container-stopped:${integrationId}:1:gitea`), "gitea stopped after being seen running");
assert.ok(ids.includes(`services:container-host:${integrationId}:2`), "Anna cannot be reached");
assert.match(report.alerts.find((x) => x.id.endsWith(`container-host:${integrationId}:2`))!.message, /can't reach the Docker host Anna — agent not connected/);
annaUp = true;
});
test("the Docker page gets each container's problem and whether it is left alone, for anyone signed in", async () => {
who = "viewer";
const body = (await (await fetch(`${base}/containers`, { headers: { connection: "close" } })).json()) as any;
const gitea = body.containers.find((x: any) => x.name === "gitea");
assert.deepEqual([gitea.problem?.kind, gitea.leftAlone], ["stopped", false]);
assert.equal(body.containers.find((x: any) => x.name === "ollama").problem, null);
assert.equal(body.alerts, true);
who = "admin";
});
test("an administrator can leave a container alone and watch it again; nobody else can, and it is in the audit log", async () => {
const put = (leaveAlone: boolean) => fetch(`${base}/leave-alone`, { method: "PUT", headers: { "content-type": "application/json", connection: "close" }, body: JSON.stringify({ environmentName: "Jackie", name: "gitea", leaveAlone }) });
who = "viewer";
assert.equal((await put(true)).status, 403);
who = "admin";
assert.equal((await put(true)).status, 200);
assert.deepEqual((await getSettings()).healthChecks.containerIgnore, ["Jackie/gitea"]);
let body = (await (await fetch(`${base}/containers`, { headers: { connection: "close" } })).json()) as any;
assert.deepEqual([body.containers.find((x: any) => x.name === "gitea").leftAlone, body.containers.find((x: any) => x.name === "gitea").problem], [true, null]);
invalidateAlertsCache();
assert.ok(!(await getAlerts(true)).alerts.some((x) => x.id.includes("container-stopped") && x.id.endsWith("gitea")), "no alert for it now");
assert.equal((await put(false)).status, 200);
assert.deepEqual((await getSettings()).healthChecks.containerIgnore, []);
const audited = await db.select().from(auditLog).where(eq(auditLog.targetType, "container"));
assert.deepEqual(audited.map((a) => [a.action, a.targetId]), [["container_leave_alone", "Jackie/gitea"], ["container_watch", "Jackie/gitea"]]);
body = (await (await fetch(`${base}/containers`, { headers: { connection: "close" } })).json()) as any;
assert.equal(body.containers.find((x: any) => x.name === "gitea").leftAlone, false);
});
test("it can be switched off, and then no container alerts are made", async () => {
const checks = (await getSettings()).healthChecks;
assert.equal(checks.containerAlerts, true, "on by default");
await updateSettings({ healthChecks: { ...checks, containerAlerts: false } });
assert.equal((await collectSnapshot()).snapshot.docker, undefined);
await updateSettings({ healthChecks: { ...checks, containerAlerts: true } });
});
});