The health pass keeps the host ports Dockhand says running containers
publish (new table server_container_ports, migration 0080), matched to a
server by its Docker host's name. They name the port on the server's Ports
card and the network Ports table, ahead of a port preset and behind a
label somebody wrote, and in the new-port alert ("8096/tcp jellyfin
(container)"). Never written onto port notes, so a container's port is
still a new port; an unreadable host keeps its names, a day-old name is
dropped. Dockhand is now read on the health pass even with container
alerts off.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
286 lines
19 KiB
TypeScript
286 lines
19 KiB
TypeScript
/**
|
|
* Containers that are broken: what Docker's status text says, which containers are left alone, when a stop is news and when it is not, what is remembered between looks, how a
|
|
* Docker host that cannot be reached is told apart from one with nothing on it, and what the alerts, the Docker page and the "leave alone" button do with it.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { createServer as createHttpServer, type Server } from "node:http";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { AddressInfo } from "node:net";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-containers-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const express = (await import("express")).default;
|
|
const { db } = await import("../src/db/client.js");
|
|
const { users, integrations } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const c = await import("../src/services/containerHealth.js");
|
|
const { evaluateHealth, collectSnapshot, runHealthCheck } = await import("../src/services/healthMonitor.js");
|
|
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
|
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
|
const { integrationsRouter } = await import("../src/routes/integrations.js");
|
|
const { getSettings, updateSettings } = await import("../src/services/settingsStore.js");
|
|
const { auditLog } = await import("../src/db/schema.js");
|
|
const { eq } = await import("drizzle-orm");
|
|
|
|
await runMigrations();
|
|
|
|
const NOW = Date.parse("2026-10-24T10:00:00Z");
|
|
const HOUR = 3_600_000;
|
|
const info = (name: string, state: string, status: string, env = "Jackie", environmentId = 1): import("../src/services/containerHealth.js").ContainerInfo => ({ environmentId, environmentName: env, name, state, status });
|
|
|
|
describe("what Docker's status text says", () => {
|
|
test("unhealthy, an exit code, and how long ago", () => {
|
|
assert.deepEqual(c.parseStatus("Up 6 days (unhealthy)"), { unhealthy: true, exitCode: null, agoMs: null });
|
|
assert.deepEqual(c.parseStatus("Up 6 days (healthy)"), { unhealthy: false, exitCode: null, agoMs: null });
|
|
assert.deepEqual(c.parseStatus("Restarting (127) 39 seconds ago"), { unhealthy: false, exitCode: 127, agoMs: 39_000 });
|
|
assert.deepEqual(c.parseStatus("Exited (0) 15 minutes ago"), { unhealthy: false, exitCode: 0, agoMs: 15 * 60_000 });
|
|
assert.equal(c.parseStatus("Exited (2) 3 months ago").agoMs, 3 * 2_592_000_000);
|
|
assert.equal(c.parseStatus("Exited (0) About an hour ago").agoMs, HOUR);
|
|
assert.equal(c.parseStatus("Exited (0) Less than a second ago").agoMs, 1000);
|
|
assert.equal(c.parseStatus("Exited (137) 2 weeks ago").exitCode, 137);
|
|
assert.deepEqual(c.parseStatus("Created"), { unhealthy: false, exitCode: null, agoMs: null });
|
|
});
|
|
|
|
test("names to leave alone: exact, with a host, with a *, and the scanners Dockhand starts", () => {
|
|
const sub = info("planka-planka-1", "exited", "Exited (0) 2 weeks ago", "Siri");
|
|
assert.equal(c.isLeftAlone(sub, []), false);
|
|
assert.equal(c.isLeftAlone(sub, ["planka-planka-1"]), true);
|
|
assert.equal(c.isLeftAlone(sub, ["Siri/planka-planka-1"]), true);
|
|
assert.equal(c.isLeftAlone(sub, ["Jackie/planka-planka-1"]), false, "another host's");
|
|
assert.equal(c.isLeftAlone(sub, ["planka-*"]), true);
|
|
assert.equal(c.isLeftAlone(sub, ["PLANKA-*"]), true, "capitals don't matter");
|
|
assert.equal(c.isLeftAlone(sub, ["planka.planka"]), false, "a dot is a dot");
|
|
assert.equal(c.isLeftAlone(info("dockhand-grype-1792611378733-7036sl", "created", "Created"), []), true);
|
|
assert.equal(c.isLeftAlone(info("dockhand-trivy-1792847558773-pu9ivk", "running", "Up 25 seconds"), []), true);
|
|
});
|
|
});
|
|
|
|
describe("one container at a time", () => {
|
|
const judge = (x: ReturnType<typeof info>, prev?: import("../src/services/containerHealth.js").ContainerMemory[string]) => c.judgeContainer(x, prev, NOW);
|
|
|
|
test("a restart loop is critical, and says the exit code", () => {
|
|
const j = judge(info("netbox-netbox-worker-1", "restarting", "Restarting (127) 39 seconds ago"));
|
|
assert.deepEqual([j.problem?.kind, j.problem?.severity, j.problem?.text], ["restarting", "critical", "keeps restarting (exit code 127)"]);
|
|
});
|
|
|
|
test("an unhealthy container is a warning; a healthy or plain running one is nothing", () => {
|
|
assert.equal(judge(info("drawio", "running", "Up 4 days (unhealthy)")).problem?.kind, "unhealthy");
|
|
assert.equal(judge(info("homebox", "running", "Up 5 days (healthy)")).problem, null);
|
|
assert.equal(judge(info("ollama", "running", "Up 2 days")).problem, null);
|
|
});
|
|
|
|
test("a stop is news when it was seen happen, or happened within the day; not when it has been so all along", () => {
|
|
const iso = new Date(NOW - HOUR).toISOString();
|
|
assert.equal(judge(info("grocy", "exited", "Exited (0) 15 minutes ago")).problem?.kind, "stopped", "never seen, stopped a quarter of an hour ago");
|
|
assert.equal(judge(info("planka", "exited", "Exited (0) 2 weeks ago")).problem, null, "never seen, down for weeks");
|
|
assert.equal(judge(info("vikunja", "exited", "Exited (2) 3 months ago")).problem, null);
|
|
assert.equal(judge(info("x", "exited", "Exited (0) 2 weeks ago"), { s: "up", since: iso }).problem?.kind, "stopped", "it was running at the last look");
|
|
assert.equal(judge(info("x", "exited", "Exited (0) 2 weeks ago"), { s: "down", since: iso }).problem, null, "it was already down");
|
|
assert.equal(judge(info("x", "exited", "Exited (0) 2 days ago"), { s: "down", since: iso, alert: true }).problem?.kind, "stopped", "once raised it stays");
|
|
assert.equal(judge(info("x", "exited", "Exited (2) 20 hours ago")).problem?.text, "has stopped (exit code 2)");
|
|
assert.equal(judge(info("x", "dead", "Dead"), { s: "up", since: iso }).problem?.text, "is dead");
|
|
});
|
|
|
|
test("created and never started is said only after it has been so for an hour", () => {
|
|
const first = judge(info("kestra", "created", "Created"));
|
|
assert.equal(first.problem, null);
|
|
assert.deepEqual(first.next, { s: "created", since: new Date(NOW).toISOString() });
|
|
assert.equal(judge(info("kestra", "created", "Created"), { s: "created", since: new Date(NOW - 30 * 60_000).toISOString() }).problem, null);
|
|
assert.equal(judge(info("kestra", "created", "Created"), { s: "created", since: new Date(NOW - 2 * HOUR).toISOString() }).problem?.kind, "created");
|
|
});
|
|
|
|
test("what is remembered: running is up, and a recovered one is no longer alerting", () => {
|
|
assert.deepEqual(judge(info("x", "running", "Up 1 hour"), { s: "down", since: "2026-10-23T00:00:00.000Z", alert: true }).next, { s: "up", since: new Date(NOW).toISOString() });
|
|
assert.equal(judge(info("x", "paused", "Up 1 hour (Paused)")).problem, null);
|
|
});
|
|
});
|
|
|
|
describe("a whole Docker integration, over several looks", () => {
|
|
/** A few of the containers from a real export. */
|
|
const export1 = [
|
|
info("netbox-netbox-worker-1", "restarting", "Restarting (127) 39 seconds ago"),
|
|
info("portall-socket-proxy", "running", "Up 6 days (unhealthy)"),
|
|
info("drawio", "running", "Up 4 days (unhealthy)", "Siri", 2),
|
|
info("grocy", "exited", "Exited (0) 15 minutes ago", "Gabriella", 3),
|
|
info("planka-planka-1", "exited", "Exited (0) 2 weeks ago", "Siri", 2),
|
|
info("vikunja-vikunja-1", "exited", "Exited (2) 3 months ago", "Nova", 4),
|
|
info("nextcloud-aio-watchtower", "exited", "Exited (0) 2 days ago"),
|
|
info("snipe-itnew-app-1", "created", "Created", "Nova", 4),
|
|
info("dockhand-grype-1792611378733-7036sl", "created", "Created", "Anna", 5),
|
|
info("gitea", "running", "Up 4 days"),
|
|
];
|
|
const snap = (containers = export1, unreachable: import("../src/services/containerHealth.js").DockerSnapshot["unreachable"] = []): import("../src/services/containerHealth.js").DockerSnapshot => ({ integrationId: 7, integrationName: "Dockhand", containers, unreachable });
|
|
|
|
test("the first look finds what is wrong now, and nothing about what has long been stopped", () => {
|
|
const { conditions, next } = c.evaluateContainers([snap()], {}, NOW, []);
|
|
assert.deepEqual(conditions.map((x) => x.key).sort(), [
|
|
"container-restarting:7:1:netbox-netbox-worker-1",
|
|
"container-stopped:7:3:grocy",
|
|
"container-unhealthy:7:1:portall-socket-proxy",
|
|
"container-unhealthy:7:2:drawio",
|
|
]);
|
|
const worker = conditions.find((x) => x.key.startsWith("container-restarting"))!;
|
|
assert.deepEqual([worker.severity, worker.source, worker.message], ["critical", "dockhand:7:1", "netbox-netbox-worker-1 [Jackie] keeps restarting (exit code 127)"]);
|
|
assert.equal(conditions.find((x) => x.key.includes("grocy"))!.message, "grocy [Gabriella] has stopped");
|
|
assert.ok(next["7:4:vikunja-vikunja-1"] && !next["7:4:vikunja-vikunja-1"].alert, "remembered as down, quietly");
|
|
assert.equal(next["7:5:dockhand-grype-1792611378733-7036sl"], undefined, "the scanner is not remembered at all");
|
|
});
|
|
|
|
test("a stop that was raised stays raised, a container that starts again clears, and one that is left alone is not said", () => {
|
|
const first = c.evaluateContainers([snap()], {}, NOW, []);
|
|
const later = c.evaluateContainers([snap()], first.next, NOW + 2 * 24 * HOUR, []);
|
|
assert.ok(later.conditions.some((x) => x.key.includes("grocy")), "two days on, grocy is still stopped and still said");
|
|
assert.ok(!later.conditions.some((x) => x.key.includes("planka")), "planka never was");
|
|
const grocyBack = export1.map((x) => (x.name === "grocy" ? info("grocy", "running", "Up 3 minutes", "Gabriella", 3) : x));
|
|
const cleared = c.evaluateContainers([snap(grocyBack)], later.next, NOW + 3 * 24 * HOUR, []);
|
|
assert.ok(!cleared.conditions.some((x) => x.key.includes("grocy")));
|
|
const stoppedAgain = export1.map((x) => (x.name === "gitea" ? info("gitea", "exited", "Exited (1) 5 minutes ago") : x));
|
|
const caught = c.evaluateContainers([snap(stoppedAgain)], first.next, NOW + 15 * 60_000, []);
|
|
assert.equal(caught.conditions.find((x) => x.key.includes("gitea"))?.message, "gitea [Jackie] has stopped (exit code 1)", "running at the last look, stopped now");
|
|
const quiet = c.evaluateContainers([snap()], {}, NOW, ["Siri/drawio", "portall-*", "netbox-netbox-worker-1", "grocy"]);
|
|
assert.deepEqual(quiet.conditions, []);
|
|
});
|
|
|
|
test("a Docker host that cannot be read is said, and its containers are neither judged nor forgotten", () => {
|
|
const first = c.evaluateContainers([snap()], {}, NOW, []);
|
|
const down = c.evaluateContainers([snap(export1.filter((x) => x.environmentId !== 1), [{ environmentId: 1, environmentName: "Jackie", error: "connect ECONNREFUSED" }])], first.next, NOW + HOUR, []);
|
|
assert.ok(down.conditions.some((x) => x.key === "container-host:7:1" && x.message === "Dockhand: can't reach the Docker host Jackie — connect ECONNREFUSED"));
|
|
assert.ok(!down.conditions.some((x) => x.key === "container-restarting:7:1:netbox-netbox-worker-1"), "not judged while the host is out of reach (the alert is held by the caller)");
|
|
assert.ok(down.next["7:1:gitea"], "and what was remembered of Jackie's containers is kept");
|
|
const gone = c.evaluateContainers([snap(export1.filter((x) => x.name !== "gitea"))], first.next, NOW + HOUR, []);
|
|
assert.equal(gone.next["7:1:gitea"], undefined, "a container that has been removed is forgotten when its host is read");
|
|
});
|
|
});
|
|
|
|
describe("through the health pass, the Alerts page and the Docker page", () => {
|
|
let dockhand: Server;
|
|
let app: Server;
|
|
let base = "";
|
|
let who = "admin";
|
|
let integrationId = 0;
|
|
let annaUp = true;
|
|
let containers: Record<string, unknown>[] = [];
|
|
|
|
const row = (name: string, state: string, status: string) => ({ id: `id-${name}`, name, image: "img", state, status });
|
|
|
|
before(async () => {
|
|
await db.insert(users).values([
|
|
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
|
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
|
]);
|
|
dockhand = createHttpServer((req, res) => {
|
|
const url = new URL(req.url ?? "", "http://x");
|
|
res.setHeader("content-type", "application/json");
|
|
if (url.pathname === "/api/environments") return void res.end(JSON.stringify([{ id: 1, name: "Jackie", connectionType: "hawser" }, { id: 2, name: "Anna", connectionType: "hawser" }]));
|
|
if (url.pathname === "/api/containers/pending-updates") return void res.end(JSON.stringify({ pendingUpdates: [] }));
|
|
if (url.pathname === "/api/containers") {
|
|
if (url.searchParams.get("env") === "2" && !annaUp) {
|
|
res.statusCode = 502;
|
|
return void res.end(JSON.stringify({ error: "agent not connected" }));
|
|
}
|
|
return void res.end(JSON.stringify(url.searchParams.get("env") === "1" ? containers : [row("ollama", "running", "Up 2 days")]));
|
|
}
|
|
res.statusCode = 404;
|
|
res.end("{}");
|
|
});
|
|
await new Promise<void>((r) => dockhand.listen(0, "127.0.0.1", r));
|
|
integrationId = (await db.insert(integrations).values({ type: "dockhand", name: "Dockhand", baseUrl: "", config: JSON.stringify({ url: `http://127.0.0.1:${(dockhand.address() as AddressInfo).port}`, token: "t" }) }).returning({ id: integrations.id }))[0].id;
|
|
const a = express();
|
|
a.use(express.json());
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
|
next();
|
|
});
|
|
a.use("/api/integrations", integrationsRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/integrations/${integrationId}/dockhand`;
|
|
});
|
|
after(() => {
|
|
dockhand.close();
|
|
app.close();
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows keeps the database file locked until the process exits.
|
|
}
|
|
});
|
|
|
|
test("the health pass reads Dockhand and finds the broken ones; the Alerts page says them under Docker, with a link", async () => {
|
|
containers = [row("netbox-worker", "restarting", "Restarting (127) 39 seconds ago"), row("gitea", "running", "Up 4 days"), row("drawio", "running", "Up 4 days (unhealthy)")];
|
|
const { snapshot } = await collectSnapshot();
|
|
assert.deepEqual(snapshot.docker?.map((d) => [d.integrationName, d.containers.length, d.unreachable.length]), [["Dockhand", 4, 0]]);
|
|
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
|
assert.deepEqual(list.filter((x) => x.key.startsWith("container-")).map((x) => x.key).sort(), [`container-restarting:${integrationId}:1:netbox-worker`, `container-unhealthy:${integrationId}:1:drawio`]);
|
|
assert.equal(subjectOfConditionKey(`container-restarting:${integrationId}:1:netbox-worker`), `integration:${integrationId}`, "maintenance on the integration holds it");
|
|
|
|
invalidateAlertsCache();
|
|
const report = await getAlerts(true);
|
|
const a = report.alerts.find((x) => x.id === `services:container-restarting:${integrationId}:1:netbox-worker`)!;
|
|
assert.ok(a);
|
|
assert.deepEqual([a.severity, a.category, a.source, a.link], ["critical", "services", "Docker", "/docker"]);
|
|
});
|
|
|
|
test("a pass remembers what it saw, so a container that stops afterwards is caught; and an unreachable host is said and held", async () => {
|
|
await runHealthCheck(Date.now());
|
|
const memory = await c.loadContainerMemory();
|
|
assert.equal(memory[`${integrationId}:1:gitea`]?.s, "up");
|
|
containers = containers.map((x) => (x.name === "gitea" ? row("gitea", "exited", "Exited (1) 2 minutes ago") : x));
|
|
annaUp = false;
|
|
invalidateAlertsCache();
|
|
const report = await getAlerts(true);
|
|
const ids = report.alerts.map((x) => x.id);
|
|
assert.ok(ids.includes(`services:container-stopped:${integrationId}:1:gitea`), "gitea stopped after being seen running");
|
|
assert.ok(ids.includes(`services:container-host:${integrationId}:2`), "Anna cannot be reached");
|
|
assert.match(report.alerts.find((x) => x.id.endsWith(`container-host:${integrationId}:2`))!.message, /can't reach the Docker host Anna — agent not connected/);
|
|
annaUp = true;
|
|
});
|
|
|
|
test("the Docker page gets each container's problem and whether it is left alone, for anyone signed in", async () => {
|
|
who = "viewer";
|
|
const body = (await (await fetch(`${base}/containers`, { headers: { connection: "close" } })).json()) as any;
|
|
const gitea = body.containers.find((x: any) => x.name === "gitea");
|
|
assert.deepEqual([gitea.problem?.kind, gitea.leftAlone], ["stopped", false]);
|
|
assert.equal(body.containers.find((x: any) => x.name === "ollama").problem, null);
|
|
assert.equal(body.alerts, true);
|
|
who = "admin";
|
|
});
|
|
|
|
test("an administrator can leave a container alone and watch it again; nobody else can, and it is in the audit log", async () => {
|
|
const put = (leaveAlone: boolean) => fetch(`${base}/leave-alone`, { method: "PUT", headers: { "content-type": "application/json", connection: "close" }, body: JSON.stringify({ environmentName: "Jackie", name: "gitea", leaveAlone }) });
|
|
who = "viewer";
|
|
assert.equal((await put(true)).status, 403);
|
|
who = "admin";
|
|
assert.equal((await put(true)).status, 200);
|
|
assert.deepEqual((await getSettings()).healthChecks.containerIgnore, ["Jackie/gitea"]);
|
|
let body = (await (await fetch(`${base}/containers`, { headers: { connection: "close" } })).json()) as any;
|
|
assert.deepEqual([body.containers.find((x: any) => x.name === "gitea").leftAlone, body.containers.find((x: any) => x.name === "gitea").problem], [true, null]);
|
|
invalidateAlertsCache();
|
|
assert.ok(!(await getAlerts(true)).alerts.some((x) => x.id.includes("container-stopped") && x.id.endsWith("gitea")), "no alert for it now");
|
|
assert.equal((await put(false)).status, 200);
|
|
assert.deepEqual((await getSettings()).healthChecks.containerIgnore, []);
|
|
const audited = await db.select().from(auditLog).where(eq(auditLog.targetType, "container"));
|
|
assert.deepEqual(audited.map((a) => [a.action, a.targetId]), [["container_leave_alone", "Jackie/gitea"], ["container_watch", "Jackie/gitea"]]);
|
|
body = (await (await fetch(`${base}/containers`, { headers: { connection: "close" } })).json()) as any;
|
|
assert.equal(body.containers.find((x: any) => x.name === "gitea").leftAlone, false);
|
|
});
|
|
|
|
test("it can be switched off, and then no container alerts are made", async () => {
|
|
const checks = (await getSettings()).healthChecks;
|
|
assert.equal(checks.containerAlerts, true, "on by default");
|
|
await updateSettings({ healthChecks: { ...checks, containerAlerts: false } });
|
|
assert.equal((await collectSnapshot()).snapshot.docker, undefined);
|
|
await updateSettings({ healthChecks: { ...checks, containerAlerts: true } });
|
|
});
|
|
});
|