/** * Containers that are broken: what Docker's status text says, which containers are left alone, when a stop is news and when it is not, what is remembered between looks, how a * Docker host that cannot be reached is told apart from one with nothing on it, and what the alerts, the Docker page and the "leave alone" button do with it. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { createServer as createHttpServer, type Server } from "node:http"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-containers-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, integrations } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const c = await import("../src/services/containerHealth.js"); const { evaluateHealth, collectSnapshot, runHealthCheck } = await import("../src/services/healthMonitor.js"); const { subjectOfConditionKey } = await import("../src/services/maintenance.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { integrationsRouter } = await import("../src/routes/integrations.js"); const { getSettings, updateSettings } = await import("../src/services/settingsStore.js"); const { auditLog } = await import("../src/db/schema.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); const NOW = Date.parse("2026-10-24T10:00:00Z"); const HOUR = 3_600_000; const info = (name: string, state: string, status: string, env = "Jackie", environmentId = 1): import("../src/services/containerHealth.js").ContainerInfo => ({ environmentId, environmentName: env, name, state, status }); describe("what Docker's status text says", () => { test("unhealthy, an exit code, and how long ago", () => { assert.deepEqual(c.parseStatus("Up 6 days (unhealthy)"), { unhealthy: true, exitCode: null, agoMs: null }); assert.deepEqual(c.parseStatus("Up 6 days (healthy)"), { unhealthy: false, exitCode: null, agoMs: null }); assert.deepEqual(c.parseStatus("Restarting (127) 39 seconds ago"), { unhealthy: false, exitCode: 127, agoMs: 39_000 }); assert.deepEqual(c.parseStatus("Exited (0) 15 minutes ago"), { unhealthy: false, exitCode: 0, agoMs: 15 * 60_000 }); assert.equal(c.parseStatus("Exited (2) 3 months ago").agoMs, 3 * 2_592_000_000); assert.equal(c.parseStatus("Exited (0) About an hour ago").agoMs, HOUR); assert.equal(c.parseStatus("Exited (0) Less than a second ago").agoMs, 1000); assert.equal(c.parseStatus("Exited (137) 2 weeks ago").exitCode, 137); assert.deepEqual(c.parseStatus("Created"), { unhealthy: false, exitCode: null, agoMs: null }); }); test("names to leave alone: exact, with a host, with a *, and the scanners Dockhand starts", () => { const sub = info("planka-planka-1", "exited", "Exited (0) 2 weeks ago", "Siri"); assert.equal(c.isLeftAlone(sub, []), false); assert.equal(c.isLeftAlone(sub, ["planka-planka-1"]), true); assert.equal(c.isLeftAlone(sub, ["Siri/planka-planka-1"]), true); assert.equal(c.isLeftAlone(sub, ["Jackie/planka-planka-1"]), false, "another host's"); assert.equal(c.isLeftAlone(sub, ["planka-*"]), true); assert.equal(c.isLeftAlone(sub, ["PLANKA-*"]), true, "capitals don't matter"); assert.equal(c.isLeftAlone(sub, ["planka.planka"]), false, "a dot is a dot"); assert.equal(c.isLeftAlone(info("dockhand-grype-1792611378733-7036sl", "created", "Created"), []), true); assert.equal(c.isLeftAlone(info("dockhand-trivy-1792847558773-pu9ivk", "running", "Up 25 seconds"), []), true); }); }); describe("one container at a time", () => { const judge = (x: ReturnType, prev?: import("../src/services/containerHealth.js").ContainerMemory[string]) => c.judgeContainer(x, prev, NOW); test("a restart loop is critical, and says the exit code", () => { const j = judge(info("netbox-netbox-worker-1", "restarting", "Restarting (127) 39 seconds ago")); assert.deepEqual([j.problem?.kind, j.problem?.severity, j.problem?.text], ["restarting", "critical", "keeps restarting (exit code 127)"]); }); test("an unhealthy container is a warning; a healthy or plain running one is nothing", () => { assert.equal(judge(info("drawio", "running", "Up 4 days (unhealthy)")).problem?.kind, "unhealthy"); assert.equal(judge(info("homebox", "running", "Up 5 days (healthy)")).problem, null); assert.equal(judge(info("ollama", "running", "Up 2 days")).problem, null); }); test("a stop is news when it was seen happen, or happened within the day; not when it has been so all along", () => { const iso = new Date(NOW - HOUR).toISOString(); assert.equal(judge(info("grocy", "exited", "Exited (0) 15 minutes ago")).problem?.kind, "stopped", "never seen, stopped a quarter of an hour ago"); assert.equal(judge(info("planka", "exited", "Exited (0) 2 weeks ago")).problem, null, "never seen, down for weeks"); assert.equal(judge(info("vikunja", "exited", "Exited (2) 3 months ago")).problem, null); assert.equal(judge(info("x", "exited", "Exited (0) 2 weeks ago"), { s: "up", since: iso }).problem?.kind, "stopped", "it was running at the last look"); assert.equal(judge(info("x", "exited", "Exited (0) 2 weeks ago"), { s: "down", since: iso }).problem, null, "it was already down"); assert.equal(judge(info("x", "exited", "Exited (0) 2 days ago"), { s: "down", since: iso, alert: true }).problem?.kind, "stopped", "once raised it stays"); assert.equal(judge(info("x", "exited", "Exited (2) 20 hours ago")).problem?.text, "has stopped (exit code 2)"); assert.equal(judge(info("x", "dead", "Dead"), { s: "up", since: iso }).problem?.text, "is dead"); }); test("created and never started is said only after it has been so for an hour", () => { const first = judge(info("kestra", "created", "Created")); assert.equal(first.problem, null); assert.deepEqual(first.next, { s: "created", since: new Date(NOW).toISOString() }); assert.equal(judge(info("kestra", "created", "Created"), { s: "created", since: new Date(NOW - 30 * 60_000).toISOString() }).problem, null); assert.equal(judge(info("kestra", "created", "Created"), { s: "created", since: new Date(NOW - 2 * HOUR).toISOString() }).problem?.kind, "created"); }); test("what is remembered: running is up, and a recovered one is no longer alerting", () => { assert.deepEqual(judge(info("x", "running", "Up 1 hour"), { s: "down", since: "2026-10-23T00:00:00.000Z", alert: true }).next, { s: "up", since: new Date(NOW).toISOString() }); assert.equal(judge(info("x", "paused", "Up 1 hour (Paused)")).problem, null); }); }); describe("a whole Docker integration, over several looks", () => { /** A few of the containers from a real export. */ const export1 = [ info("netbox-netbox-worker-1", "restarting", "Restarting (127) 39 seconds ago"), info("portall-socket-proxy", "running", "Up 6 days (unhealthy)"), info("drawio", "running", "Up 4 days (unhealthy)", "Siri", 2), info("grocy", "exited", "Exited (0) 15 minutes ago", "Gabriella", 3), info("planka-planka-1", "exited", "Exited (0) 2 weeks ago", "Siri", 2), info("vikunja-vikunja-1", "exited", "Exited (2) 3 months ago", "Nova", 4), info("nextcloud-aio-watchtower", "exited", "Exited (0) 2 days ago"), info("snipe-itnew-app-1", "created", "Created", "Nova", 4), info("dockhand-grype-1792611378733-7036sl", "created", "Created", "Anna", 5), info("gitea", "running", "Up 4 days"), ]; const snap = (containers = export1, unreachable: import("../src/services/containerHealth.js").DockerSnapshot["unreachable"] = []): import("../src/services/containerHealth.js").DockerSnapshot => ({ integrationId: 7, integrationName: "Dockhand", containers, unreachable }); test("the first look finds what is wrong now, and nothing about what has long been stopped", () => { const { conditions, next } = c.evaluateContainers([snap()], {}, NOW, []); assert.deepEqual(conditions.map((x) => x.key).sort(), [ "container-restarting:7:1:netbox-netbox-worker-1", "container-stopped:7:3:grocy", "container-unhealthy:7:1:portall-socket-proxy", "container-unhealthy:7:2:drawio", ]); const worker = conditions.find((x) => x.key.startsWith("container-restarting"))!; assert.deepEqual([worker.severity, worker.source, worker.message], ["critical", "dockhand:7:1", "netbox-netbox-worker-1 [Jackie] keeps restarting (exit code 127)"]); assert.equal(conditions.find((x) => x.key.includes("grocy"))!.message, "grocy [Gabriella] has stopped"); assert.ok(next["7:4:vikunja-vikunja-1"] && !next["7:4:vikunja-vikunja-1"].alert, "remembered as down, quietly"); assert.equal(next["7:5:dockhand-grype-1792611378733-7036sl"], undefined, "the scanner is not remembered at all"); }); test("a stop that was raised stays raised, a container that starts again clears, and one that is left alone is not said", () => { const first = c.evaluateContainers([snap()], {}, NOW, []); const later = c.evaluateContainers([snap()], first.next, NOW + 2 * 24 * HOUR, []); assert.ok(later.conditions.some((x) => x.key.includes("grocy")), "two days on, grocy is still stopped and still said"); assert.ok(!later.conditions.some((x) => x.key.includes("planka")), "planka never was"); const grocyBack = export1.map((x) => (x.name === "grocy" ? info("grocy", "running", "Up 3 minutes", "Gabriella", 3) : x)); const cleared = c.evaluateContainers([snap(grocyBack)], later.next, NOW + 3 * 24 * HOUR, []); assert.ok(!cleared.conditions.some((x) => x.key.includes("grocy"))); const stoppedAgain = export1.map((x) => (x.name === "gitea" ? info("gitea", "exited", "Exited (1) 5 minutes ago") : x)); const caught = c.evaluateContainers([snap(stoppedAgain)], first.next, NOW + 15 * 60_000, []); assert.equal(caught.conditions.find((x) => x.key.includes("gitea"))?.message, "gitea [Jackie] has stopped (exit code 1)", "running at the last look, stopped now"); const quiet = c.evaluateContainers([snap()], {}, NOW, ["Siri/drawio", "portall-*", "netbox-netbox-worker-1", "grocy"]); assert.deepEqual(quiet.conditions, []); }); test("a Docker host that cannot be read is said, and its containers are neither judged nor forgotten", () => { const first = c.evaluateContainers([snap()], {}, NOW, []); const down = c.evaluateContainers([snap(export1.filter((x) => x.environmentId !== 1), [{ environmentId: 1, environmentName: "Jackie", error: "connect ECONNREFUSED" }])], first.next, NOW + HOUR, []); assert.ok(down.conditions.some((x) => x.key === "container-host:7:1" && x.message === "Dockhand: can't reach the Docker host Jackie — connect ECONNREFUSED")); assert.ok(!down.conditions.some((x) => x.key === "container-restarting:7:1:netbox-netbox-worker-1"), "not judged while the host is out of reach (the alert is held by the caller)"); assert.ok(down.next["7:1:gitea"], "and what was remembered of Jackie's containers is kept"); const gone = c.evaluateContainers([snap(export1.filter((x) => x.name !== "gitea"))], first.next, NOW + HOUR, []); assert.equal(gone.next["7:1:gitea"], undefined, "a container that has been removed is forgotten when its host is read"); }); }); describe("through the health pass, the Alerts page and the Docker page", () => { let dockhand: Server; let app: Server; let base = ""; let who = "admin"; let integrationId = 0; let annaUp = true; let containers: Record[] = []; const row = (name: string, state: string, status: string) => ({ id: `id-${name}`, name, image: "img", state, status }); before(async () => { await db.insert(users).values([ { oidcSub: "admin", name: "Ada", role: "admin" }, { oidcSub: "viewer", name: "Vic", role: "viewer" }, ]); dockhand = createHttpServer((req, res) => { const url = new URL(req.url ?? "", "http://x"); res.setHeader("content-type", "application/json"); if (url.pathname === "/api/environments") return void res.end(JSON.stringify([{ id: 1, name: "Jackie", connectionType: "hawser" }, { id: 2, name: "Anna", connectionType: "hawser" }])); if (url.pathname === "/api/containers/pending-updates") return void res.end(JSON.stringify({ pendingUpdates: [] })); if (url.pathname === "/api/containers") { if (url.searchParams.get("env") === "2" && !annaUp) { res.statusCode = 502; return void res.end(JSON.stringify({ error: "agent not connected" })); } return void res.end(JSON.stringify(url.searchParams.get("env") === "1" ? containers : [row("ollama", "running", "Up 2 days")])); } res.statusCode = 404; res.end("{}"); }); await new Promise((r) => dockhand.listen(0, "127.0.0.1", r)); integrationId = (await db.insert(integrations).values({ type: "dockhand", name: "Dockhand", baseUrl: "", config: JSON.stringify({ url: `http://127.0.0.1:${(dockhand.address() as AddressInfo).port}`, token: "t" }) }).returning({ id: integrations.id }))[0].id; const a = express(); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/integrations", integrationsRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/integrations/${integrationId}/dockhand`; }); after(() => { dockhand.close(); app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("the health pass reads Dockhand and finds the broken ones; the Alerts page says them under Docker, with a link", async () => { containers = [row("netbox-worker", "restarting", "Restarting (127) 39 seconds ago"), row("gitea", "running", "Up 4 days"), row("drawio", "running", "Up 4 days (unhealthy)")]; const { snapshot } = await collectSnapshot(); assert.deepEqual(snapshot.docker?.map((d) => [d.integrationName, d.containers.length, d.unreachable.length]), [["Dockhand", 4, 0]]); const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true }); assert.deepEqual(list.filter((x) => x.key.startsWith("container-")).map((x) => x.key).sort(), [`container-restarting:${integrationId}:1:netbox-worker`, `container-unhealthy:${integrationId}:1:drawio`]); assert.equal(subjectOfConditionKey(`container-restarting:${integrationId}:1:netbox-worker`), `integration:${integrationId}`, "maintenance on the integration holds it"); invalidateAlertsCache(); const report = await getAlerts(true); const a = report.alerts.find((x) => x.id === `services:container-restarting:${integrationId}:1:netbox-worker`)!; assert.ok(a); assert.deepEqual([a.severity, a.category, a.source, a.link], ["critical", "services", "Docker", "/docker"]); }); test("a pass remembers what it saw, so a container that stops afterwards is caught; and an unreachable host is said and held", async () => { await runHealthCheck(Date.now()); const memory = await c.loadContainerMemory(); assert.equal(memory[`${integrationId}:1:gitea`]?.s, "up"); containers = containers.map((x) => (x.name === "gitea" ? row("gitea", "exited", "Exited (1) 2 minutes ago") : x)); annaUp = false; invalidateAlertsCache(); const report = await getAlerts(true); const ids = report.alerts.map((x) => x.id); assert.ok(ids.includes(`services:container-stopped:${integrationId}:1:gitea`), "gitea stopped after being seen running"); assert.ok(ids.includes(`services:container-host:${integrationId}:2`), "Anna cannot be reached"); assert.match(report.alerts.find((x) => x.id.endsWith(`container-host:${integrationId}:2`))!.message, /can't reach the Docker host Anna — agent not connected/); annaUp = true; }); test("the Docker page gets each container's problem and whether it is left alone, for anyone signed in", async () => { who = "viewer"; const body = (await (await fetch(`${base}/containers`, { headers: { connection: "close" } })).json()) as any; const gitea = body.containers.find((x: any) => x.name === "gitea"); assert.deepEqual([gitea.problem?.kind, gitea.leftAlone], ["stopped", false]); assert.equal(body.containers.find((x: any) => x.name === "ollama").problem, null); assert.equal(body.alerts, true); who = "admin"; }); test("an administrator can leave a container alone and watch it again; nobody else can, and it is in the audit log", async () => { const put = (leaveAlone: boolean) => fetch(`${base}/leave-alone`, { method: "PUT", headers: { "content-type": "application/json", connection: "close" }, body: JSON.stringify({ environmentName: "Jackie", name: "gitea", leaveAlone }) }); who = "viewer"; assert.equal((await put(true)).status, 403); who = "admin"; assert.equal((await put(true)).status, 200); assert.deepEqual((await getSettings()).healthChecks.containerIgnore, ["Jackie/gitea"]); let body = (await (await fetch(`${base}/containers`, { headers: { connection: "close" } })).json()) as any; assert.deepEqual([body.containers.find((x: any) => x.name === "gitea").leftAlone, body.containers.find((x: any) => x.name === "gitea").problem], [true, null]); invalidateAlertsCache(); assert.ok(!(await getAlerts(true)).alerts.some((x) => x.id.includes("container-stopped") && x.id.endsWith("gitea")), "no alert for it now"); assert.equal((await put(false)).status, 200); assert.deepEqual((await getSettings()).healthChecks.containerIgnore, []); const audited = await db.select().from(auditLog).where(eq(auditLog.targetType, "container")); assert.deepEqual(audited.map((a) => [a.action, a.targetId]), [["container_leave_alone", "Jackie/gitea"], ["container_watch", "Jackie/gitea"]]); body = (await (await fetch(`${base}/containers`, { headers: { connection: "close" } })).json()) as any; assert.equal(body.containers.find((x: any) => x.name === "gitea").leftAlone, false); }); test("it can be switched off, and then nothing is asked of Dockhand", async () => { const checks = (await getSettings()).healthChecks; assert.equal(checks.containerAlerts, true, "on by default"); await updateSettings({ healthChecks: { ...checks, containerAlerts: false } }); assert.equal((await collectSnapshot()).snapshot.docker, undefined); await updateSettings({ healthChecks: { ...checks, containerAlerts: true } }); }); });