Broken Docker containers are alerts

From Dockhand, on the health pass: a container that keeps restarting (critical, with
its exit code), one whose own health check fails, one that stopped unexpectedly, one
created and never started for an hour, and a Docker host Dockhand cannot reach. A stop
is only news when it was seen happen, so stacks switched off long ago stay quiet.
A Leave alone button on the Docker page and a list in Settings; on by default.

Also fixes a date-dependent storage-forecast test that pinned its clock while the code
under test reads the real one.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-24 21:03:56 +02:00
1 parent 16b24f5436
commit 0fa50c2d5a
20 files changed
+659 -17

No files matched your search

+18 -4
View File
@@ -37,9 +37,17 @@ export interface DockhandContainer {
checkedAt: string | null;
}
/** Every container Dockhand could read, and the Docker hosts it could not (so a missing host is not mistaken for a host with nothing on it). */
export interface DockhandContainerRead {
containers: DockhandContainer[];
environments: DockhandEnvironment[];
unreachable: { environmentId: number; environmentName: string; error: string }[];
}
export interface DockhandAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
listContainers(): Promise<DockhandContainer[]>;
listContainersDetailed(): Promise<DockhandContainerRead>;
startContainer(environmentId: number, containerId: string): Promise<void>;
stopContainer(environmentId: number, containerId: string): Promise<void>;
restartContainer(environmentId: number, containerId: string): Promise<void>;
@@ -84,8 +92,9 @@ export function createDockhandAdapter(config: DockhandConfig): DockhandAdapter {
}));
}
async function listContainers(): Promise<DockhandContainer[]> {
async function listContainersDetailed(): Promise<DockhandContainerRead> {
const environments = await listEnvironments();
const unreachable: DockhandContainerRead["unreachable"] = [];
const perEnv = await Promise.all(
environments.map(async (env) => {
try {
@@ -112,13 +121,18 @@ export function createDockhandAdapter(config: DockhandConfig): DockhandAdapter {
checkedAt: record?.checkedAt ?? null,
};
});
} catch {
} catch (err) {
// one unreachable host shouldn't take down the whole dashboard view
unreachable.push({ environmentId: env.id, environmentName: env.name, error: err instanceof Error ? err.message : String(err) });
return [];
}
}),
);
return perEnv.flat();
return { containers: perEnv.flat(), environments, unreachable };
}
async function listContainers(): Promise<DockhandContainer[]> {
return (await listContainersDetailed()).containers;
}
async function checkForUpdates(): Promise<{ total: number; updatesFound: number }> {
@@ -162,5 +176,5 @@ export function createDockhandAdapter(config: DockhandConfig): DockhandAdapter {
}
}
return withDiagLogging("dockhand", { ping, listContainers, startContainer, stopContainer, restartContainer, checkForUpdates });
return withDiagLogging("dockhand", { ping, listContainers, listContainersDetailed, startContainer, stopContainer, restartContainer, checkForUpdates });
}
+24 -2
View File
@@ -17,6 +17,7 @@ import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createTailscaleAdapter, isKeyExpiringSoon } from "../integrations/tailscale/adapter.js";
import { createGiteaAdapter } from "../integrations/gitea/adapter.js";
import { createDockhandAdapter } from "../integrations/dockhand/adapter.js";
import { isLeftAlone, loadContainerMemory, memoryKey, problemsFor } from "../services/containerHealth.js";
import { createSemaphoreAdapter } from "../integrations/semaphore/adapter.js";
import { createProxmoxAdapter, guestsWithoutBackupCoverage } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter } from "../integrations/synology/adapter.js";
@@ -29,6 +30,7 @@ import { removeIntegrationData, syncUnifiIntegration } from "../services/unifiSy
import { removeHomeAssistantData, syncHomeAssistant } from "../services/homeAssistantSync.js";
import { forgetUps, syncUps } from "../services/upsSync.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { ID_PATTERNS, isValidId } from "../utils/pathSegment.js";
@@ -546,9 +548,15 @@ integrationsRouter.get("/:id/dockhand/containers", asyncHandler(async (req, res)
if (!found) return;
try {
const containers = await found.adapter.listContainers();
const read = await found.adapter.listContainersDetailed();
const containers = read.containers;
// What is wrong with each, as the health pass would say it (from what it remembers; nothing is changed here), and what is left alone.
const { healthChecks } = await getSettings();
const problems = problemsFor({ integrationId: found.integration.id, integrationName: found.integration.name, containers, unreachable: read.unreachable }, await loadContainerMemory(), Date.now(), healthChecks.containerIgnore);
res.json({
containers,
containers: containers.map((c) => ({ ...c, problem: problems[memoryKey(found.integration.id, c)] ?? null, leftAlone: isLeftAlone(c, healthChecks.containerIgnore) })),
unreachable: read.unreachable,
alerts: healthChecks.containerAlerts,
summary: {
total: containers.length,
running: containers.filter((c) => c.state === "running").length,
@@ -559,6 +567,20 @@ integrationsRouter.get("/:id/dockhand/containers", asyncHandler(async (req, res)
}
}));
// Leaves a container alone — no alert for it — or watches it again. Administrators: it changes what is alerted.
integrationsRouter.put("/:id/dockhand/leave-alone", requireRole("admin"), asyncHandler(async (req, res) => {
const body = z.object({ environmentName: z.string().min(1).max(100), name: z.string().min(1).max(100), leaveAlone: z.boolean() }).safeParse(req.body);
if (!body.success) return res.status(400).json({ error: "invalid_body" });
const entry = `${body.data.environmentName}/${body.data.name}`;
const { healthChecks } = await getSettings();
const list = healthChecks.containerIgnore.filter((p) => p !== entry && p !== body.data.name);
if (body.data.leaveAlone) list.push(entry);
await updateSettings({ healthChecks: { ...healthChecks, containerIgnore: list.slice(0, 200) } });
await recordAudit({ actor: req.currentUser!, category: "settings", action: body.data.leaveAlone ? "container_leave_alone" : "container_watch", targetType: "container", targetId: entry });
invalidateAlertsCache();
res.json({ containerIgnore: list });
}));
integrationsRouter.post(
"/:id/dockhand/check-updates",
requireRole("operator"),
+1 -1
View File
@@ -144,7 +144,7 @@ const updateSchema = z.object({
.partial()
.optional(),
healthChecks: z
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365), baselineAlerts: z.boolean() })
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365), baselineAlerts: z.boolean(), containerAlerts: z.boolean(), containerIgnore: z.array(z.string().trim().min(1).max(100)).max(200) })
.partial()
.optional(),
quietHours: z
+4 -2
View File
@@ -108,6 +108,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "ups", id: Number(m[1]), category: "monitoring" };
m = /^slo:(\d+)$/.exec(key);
if (m) return { type: "slo", id: Number(m[1]), category: "monitoring" };
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
return { type: "server", id: null, category: "disk" };
}
@@ -230,7 +232,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : undefined,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
@@ -239,7 +241,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi|ups):(\d+)(?::(.+))?$/.exec(h);
const m = /^(proxmox|synology|unifi|ups|dockhand):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
+163
View File
@@ -0,0 +1,163 @@
/**
* Containers that are broken, said before somebody notices the app behind them is gone: one that keeps restarting, one whose own health check says it is unhealthy, one that
* stopped without anyone meaning it to, one that was created and never started, and a Docker host Dockhand can no longer reach. Read from Dockhand (the Docker integration) on
* the health pass; nothing is sent to Docker.
*
* The hard part is *stopped*. Plenty of containers are stopped on purpose — a stack that is switched off, a one-shot job, a watchtower that ran and exited — and nobody wants an
* alert for each of them for ever. So a stop is only news when it was seen happen: the container was running at the previous look, or (for one the app has never seen) it
* stopped within the last day, as Docker's own status text says. One that has been down all along is left alone, and so is anything named in the list of containers to leave alone.
* An alert, once raised, stays until the container runs again, is removed, or is left alone.
*/
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export interface ContainerInfo {
environmentId: number;
environmentName: string;
name: string;
state: string;
status: string;
}
export interface DockerSnapshot {
integrationId: number;
integrationName: string;
containers: ContainerInfo[];
/** Docker hosts Dockhand knows but could not read this time. */
unreachable: { environmentId: number; environmentName: string; error: string }[];
}
/** What was seen of each container, by "integration:environment:name", kept between looks. */
export type ContainerMemory = Record<string, { s: "up" | "down" | "created"; since: string; alert?: boolean }>;
export interface ContainerCondition {
key: string;
source: string;
message: string;
severity: "critical" | "warning";
}
/** A container created and never started is left a while (a stack is being brought up) before it is said. */
export const CREATED_GRACE_MS = 60 * 60_000;
/** A container the app has never seen running counts as "just stopped" if Docker says it stopped within this long. */
export const RECENT_STOP_MS = 24 * 3_600_000;
/** Containers Dockhand starts to scan images with, which come and go. Always left alone. */
export const BUILT_IN_IGNORE = ["dockhand-grype-*", "dockhand-trivy-*"];
const UNITS: Record<string, number> = { second: 1_000, minute: 60_000, hour: 3_600_000, day: 86_400_000, week: 604_800_000, month: 2_592_000_000, year: 31_536_000_000 };
export interface ParsedStatus {
unhealthy: boolean;
/** The exit code in "Exited (127)" or "Restarting (127)". */
exitCode: number | null;
/** How long ago it exited, from "… 15 minutes ago"; null when the text does not say. */
agoMs: number | null;
}
/** Pure: what Docker's human status text says ("Up 6 days (unhealthy)", "Exited (0) 15 minutes ago", "Restarting (127) 39 seconds ago"). */
export function parseStatus(status: string): ParsedStatus {
const code = /^(?:Exited|Restarting)\s*\((\d+)\)/i.exec(status);
const ago = /(?:(\d+)|an?|about an?|less than a)\s+(second|minute|hour|day|week|month|year)s?\s+ago/i.exec(status);
return {
unhealthy: /\(unhealthy\)/i.test(status),
exitCode: code ? Number(code[1]) : null,
agoMs: ago ? (ago[1] ? Number(ago[1]) : 1) * UNITS[ago[2].toLowerCase()] : null,
};
}
/** Pure: whether a container is named in the list of ones to leave alone — by name, or "host/name"; * stands for anything. */
export function isLeftAlone(c: Pick<ContainerInfo, "name" | "environmentName">, patterns: string[]): boolean {
const match = (pattern: string, text: string) => new RegExp(`^${pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*")}$`, "i").test(text);
return [...BUILT_IN_IGNORE, ...patterns].some((p) => match(p.trim(), c.name) || match(p.trim(), `${c.environmentName}/${c.name}`));
}
export const memoryKey = (integrationId: number, c: Pick<ContainerInfo, "environmentId" | "name">) => `${integrationId}:${c.environmentId}:${c.name}`;
export type ProblemKind = "restarting" | "unhealthy" | "stopped" | "created";
export interface ContainerProblem {
kind: ProblemKind;
text: string;
severity: "critical" | "warning";
}
/** Pure: what is wrong with one container, given what was seen of it before (or nothing). Also what to remember of it. */
export function judgeContainer(c: ContainerInfo, prev: ContainerMemory[string] | undefined, nowMs: number): { problem: ContainerProblem | null; next: ContainerMemory[string] } {
const nowIso = new Date(nowMs).toISOString();
const st = parseStatus(c.status);
if (c.state === "restarting" || /^Restarting/i.test(c.status)) {
return { problem: { kind: "restarting", text: `keeps restarting${st.exitCode !== null ? ` (exit code ${st.exitCode})` : ""}`, severity: "critical" }, next: { s: "up", since: prev?.since ?? nowIso } };
}
if (c.state === "running") {
return { problem: st.unhealthy ? { kind: "unhealthy", text: "is unhealthy — its own health check is failing", severity: "warning" } : null, next: { s: "up", since: prev?.s === "up" ? prev.since : nowIso } };
}
if (c.state === "created") {
const since = prev?.s === "created" ? prev.since : nowIso;
const long = nowMs - Date.parse(since) >= CREATED_GRACE_MS;
return { problem: long ? { kind: "created", text: "was created but has never been started", severity: "warning" } : null, next: { s: "created", since } };
}
if (c.state === "exited" || c.state === "dead") {
const wasUp = prev?.s === "up";
const justStopped = !prev && st.agoMs !== null && st.agoMs < RECENT_STOP_MS;
const alert = prev?.alert === true || wasUp || justStopped;
const text = c.state === "dead" ? "is dead" : `has stopped${st.exitCode ? ` (exit code ${st.exitCode})` : ""}`;
return { problem: alert ? { kind: "stopped", text, severity: "warning" } : null, next: { s: "down", since: prev?.s === "down" ? prev.since : nowIso, ...(alert ? { alert: true } : {}) } };
}
// paused and anything else: not judged
return { problem: null, next: { s: "down", since: prev?.s === "down" ? prev.since : nowIso } };
}
/** Pure: the conditions for what was read, and what to remember. Reads `memory`, changes nothing. */
export function evaluateContainers(snapshots: DockerSnapshot[], memory: ContainerMemory, nowMs: number, ignore: string[]): { conditions: ContainerCondition[]; next: ContainerMemory } {
const conditions: ContainerCondition[] = [];
const next: ContainerMemory = { ...memory };
for (const snap of snapshots) {
const unreachable = new Set(snap.unreachable.map((u) => u.environmentId));
// What is remembered of the hosts that were read is rebuilt from what was read; a host that could not be read keeps what it had.
for (const k of Object.keys(next)) {
const [int, env] = k.split(":");
if (Number(int) === snap.integrationId && !unreachable.has(Number(env))) delete next[k];
}
for (const u of snap.unreachable) {
conditions.push({ key: `container-host:${snap.integrationId}:${u.environmentId}`, source: `dockhand:${snap.integrationId}`, severity: "warning", message: `${snap.integrationName}: can't reach the Docker host ${u.environmentName} — ${u.error}` });
}
for (const c of snap.containers) {
if (unreachable.has(c.environmentId) || isLeftAlone(c, ignore)) continue;
const key = memoryKey(snap.integrationId, c);
const { problem, next: remembered } = judgeContainer(c, memory[key], nowMs);
next[key] = remembered;
if (!problem) continue;
conditions.push({ key: `container-${problem.kind}:${snap.integrationId}:${c.environmentId}:${c.name}`, source: `dockhand:${snap.integrationId}:${c.environmentId}`, severity: problem.severity, message: `${c.name} [${c.environmentName}] ${problem.text}` });
}
}
return { conditions, next };
}
const MEMORY_FLAG = "containerMemory";
export async function loadContainerMemory(): Promise<ContainerMemory> {
try {
const raw = await getInternalFlag(MEMORY_FLAG);
return raw ? (JSON.parse(raw) as ContainerMemory) : {};
} catch {
return {};
}
}
/** Keeps what was seen, for what is still wanted (integrations that no longer exist are dropped). */
export async function saveContainerMemory(next: ContainerMemory, integrationIds: number[]): Promise<void> {
const wanted = new Set(integrationIds);
const kept: ContainerMemory = {};
for (const [k, v] of Object.entries(next)) if (wanted.has(Number(k.split(":")[0]))) kept[k] = v;
await setInternalFlag(MEMORY_FLAG, JSON.stringify(kept));
}
/** What is wrong with each container, by memory key, for the Docker page (reads, never changes what is remembered). */
export function problemsFor(snap: DockerSnapshot, memory: ContainerMemory, nowMs: number, ignore: string[]): Record<string, ContainerProblem> {
const out: Record<string, ContainerProblem> = {};
for (const c of snap.containers) {
if (isLeftAlone(c, ignore)) continue;
const { problem } = judgeContainer(c, memory[memoryKey(snap.integrationId, c)], nowMs);
if (problem) out[memoryKey(snap.integrationId, c)] = problem;
}
return out;
}
+43 -1
View File
@@ -1,4 +1,4 @@
import { eq } from "drizzle-orm";
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
@@ -18,6 +18,8 @@ import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThres
import { computeRuleFindings, findingKey, listRules, type RuleFinding } from "./alertRules.js";
import { baselineKey, computeBaselineFindings, type BaselineFinding } from "./baselines.js";
import { levelFindings, levelKey, type LevelFinding } from "./serviceLevels.js";
import { createDockhandAdapter } from "../integrations/dockhand/adapter.js";
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
import { recordStorageSamples } from "./storageHistory.js";
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
@@ -84,6 +86,13 @@ export interface HealthSnapshot {
baselines?: BaselineFinding[];
/** Service-level targets whose error budget for the month is mostly or wholly used (services/serviceLevels.ts). */
levels?: LevelFinding[];
/** Each Docker integration's containers as last read, with what was remembered of them (services/containerHealth.ts). */
docker?: DockerSnapshot[];
dockerMemory?: ContainerMemory;
/** Containers to leave alone. */
dockerIgnore?: string[];
/** The clock the container judgements are made on (the pass's, so a test can set it). */
dockerNow?: number;
}
export interface Thresholds {
serverOfflineMinutes: number;
@@ -186,6 +195,11 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
}
}
// Containers that are broken, stopped unexpectedly or never started, and Docker hosts that cannot be reached.
if (snapshot.docker && snapshot.docker.length > 0) {
for (const c of evaluateContainers(snapshot.docker, snapshot.dockerMemory ?? {}, snapshot.dockerNow ?? now, snapshot.dockerIgnore ?? []).conditions) out.push(c);
}
// A service-level target that has used most of its month's error budget, or all of it.
for (const f of snapshot.levels ?? []) out.push({ key: levelKey(f), source: "slo", message: f.message, severity: f.severity });
@@ -408,6 +422,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
}
}
// Docker containers, from Dockhand: what is broken, what stopped unexpectedly. A Dockhand that cannot be read holds its containers' alerts rather than clearing them.
if (checks.containerAlerts) {
snapshot.dockerIgnore = checks.containerIgnore;
snapshot.dockerMemory = await loadContainerMemory();
for (const row of await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "dockhand"), eq(integrations.enabled, true)))) {
try {
const loaded = await loadIntegrationConfig(row.id);
if (!loaded) continue;
const read = await createDockhandAdapter(loaded.config as any).listContainersDetailed();
(snapshot.docker ??= []).push({ integrationId: row.id, integrationName: row.name, containers: read.containers, unreachable: read.unreachable });
for (const u of read.unreachable) held.add(`dockhand:${row.id}:${u.environmentId}`);
} catch (err) {
console.error(`[health] couldn't read Docker integration ${row.id}:`, err instanceof Error ? err.message : err);
held.add(`dockhand:${row.id}`);
}
}
}
// Service-level targets: how much of the month's error budget is used. A failure here costs only this check.
try {
snapshot.levels = await levelFindings();
@@ -550,6 +582,16 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added:
const { added: announced, resolved, next, newlyDown, ended } = diffConditions(await loadState(), current, held, await activeSubjects(new Date(now)), coveredKeys);
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
// What was seen of the containers is kept for the next look (only this pass writes it; the Alerts page only reads it).
if (snapshot.docker) {
try {
const after = evaluateContainers(snapshot.docker, snapshot.dockerMemory ?? {}, now, snapshot.dockerIgnore ?? []);
await saveContainerMemory(after.next, (await db.select({ id: integrations.id }).from(integrations).where(eq(integrations.type, "dockhand"))).map((r) => r.id));
} catch (err) {
console.error("[health] couldn't keep what was seen of the containers:", err);
}
}
// A host's message says what else it takes with it.
const added = announced.map((c) => {
const m = /^offline:(server|device):(\d+)$/.exec(c.key);
+2
View File
@@ -29,6 +29,8 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `server:${m[1]}`;
m = /^baseline:(?:cpu|mem):server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^offline:device:(\d+)$/.exec(key);
+5 -1
View File
@@ -140,6 +140,10 @@ export interface HealthCheckSettings {
osSupportWarnDays: number;
/** A server whose CPU load or memory use over the last hour is clearly higher than is normal for this hour of the week is a warning (services/baselines.ts). */
baselineAlerts: boolean;
/** A container that keeps restarting, is unhealthy, stopped unexpectedly or never started, and a Docker host Dockhand cannot reach, is an alert (services/containerHealth.ts). */
containerAlerts: boolean;
/** Containers to leave alone: a name, or "host/name"; * stands for anything. */
containerIgnore: string[];
}
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
@@ -323,7 +327,7 @@ const DEFAULTS: AppSettings = {
selfBackup: { enabled: true, time: "03:30", keep: 14 },
profilePictures: { gravatar: false, identityProvider: false },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, containerIgnore: [] },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
consistency: { excludedRanges: ["172.16.0.0/12"] },
+285
View File
@@ -0,0 +1,285 @@
/**
* Containers that are broken: what Docker's status text says, which containers are left alone, when a stop is news and when it is not, what is remembered between looks, how a
* Docker host that cannot be reached is told apart from one with nothing on it, and what the alerts, the Docker page and the "leave alone" button do with it.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-containers-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const c = await import("../src/services/containerHealth.js");
const { evaluateHealth, collectSnapshot, runHealthCheck } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { integrationsRouter } = await import("../src/routes/integrations.js");
const { getSettings, updateSettings } = await import("../src/services/settingsStore.js");
const { auditLog } = await import("../src/db/schema.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const NOW = Date.parse("2026-10-24T10:00:00Z");
const HOUR = 3_600_000;
const info = (name: string, state: string, status: string, env = "Jackie", environmentId = 1): import("../src/services/containerHealth.js").ContainerInfo => ({ environmentId, environmentName: env, name, state, status });
describe("what Docker's status text says", () => {
test("unhealthy, an exit code, and how long ago", () => {
assert.deepEqual(c.parseStatus("Up 6 days (unhealthy)"), { unhealthy: true, exitCode: null, agoMs: null });
assert.deepEqual(c.parseStatus("Up 6 days (healthy)"), { unhealthy: false, exitCode: null, agoMs: null });
assert.deepEqual(c.parseStatus("Restarting (127) 39 seconds ago"), { unhealthy: false, exitCode: 127, agoMs: 39_000 });
assert.deepEqual(c.parseStatus("Exited (0) 15 minutes ago"), { unhealthy: false, exitCode: 0, agoMs: 15 * 60_000 });
assert.equal(c.parseStatus("Exited (2) 3 months ago").agoMs, 3 * 2_592_000_000);
assert.equal(c.parseStatus("Exited (0) About an hour ago").agoMs, HOUR);
assert.equal(c.parseStatus("Exited (0) Less than a second ago").agoMs, 1000);
assert.equal(c.parseStatus("Exited (137) 2 weeks ago").exitCode, 137);
assert.deepEqual(c.parseStatus("Created"), { unhealthy: false, exitCode: null, agoMs: null });
});
test("names to leave alone: exact, with a host, with a *, and the scanners Dockhand starts", () => {
const sub = info("planka-planka-1", "exited", "Exited (0) 2 weeks ago", "Siri");
assert.equal(c.isLeftAlone(sub, []), false);
assert.equal(c.isLeftAlone(sub, ["planka-planka-1"]), true);
assert.equal(c.isLeftAlone(sub, ["Siri/planka-planka-1"]), true);
assert.equal(c.isLeftAlone(sub, ["Jackie/planka-planka-1"]), false, "another host's");
assert.equal(c.isLeftAlone(sub, ["planka-*"]), true);
assert.equal(c.isLeftAlone(sub, ["PLANKA-*"]), true, "capitals don't matter");
assert.equal(c.isLeftAlone(sub, ["planka.planka"]), false, "a dot is a dot");
assert.equal(c.isLeftAlone(info("dockhand-grype-1792611378733-7036sl", "created", "Created"), []), true);
assert.equal(c.isLeftAlone(info("dockhand-trivy-1792847558773-pu9ivk", "running", "Up 25 seconds"), []), true);
});
});
describe("one container at a time", () => {
const judge = (x: ReturnType<typeof info>, prev?: import("../src/services/containerHealth.js").ContainerMemory[string]) => c.judgeContainer(x, prev, NOW);
test("a restart loop is critical, and says the exit code", () => {
const j = judge(info("netbox-netbox-worker-1", "restarting", "Restarting (127) 39 seconds ago"));
assert.deepEqual([j.problem?.kind, j.problem?.severity, j.problem?.text], ["restarting", "critical", "keeps restarting (exit code 127)"]);
});
test("an unhealthy container is a warning; a healthy or plain running one is nothing", () => {
assert.equal(judge(info("drawio", "running", "Up 4 days (unhealthy)")).problem?.kind, "unhealthy");
assert.equal(judge(info("homebox", "running", "Up 5 days (healthy)")).problem, null);
assert.equal(judge(info("ollama", "running", "Up 2 days")).problem, null);
});
test("a stop is news when it was seen happen, or happened within the day; not when it has been so all along", () => {
const iso = new Date(NOW - HOUR).toISOString();
assert.equal(judge(info("grocy", "exited", "Exited (0) 15 minutes ago")).problem?.kind, "stopped", "never seen, stopped a quarter of an hour ago");
assert.equal(judge(info("planka", "exited", "Exited (0) 2 weeks ago")).problem, null, "never seen, down for weeks");
assert.equal(judge(info("vikunja", "exited", "Exited (2) 3 months ago")).problem, null);
assert.equal(judge(info("x", "exited", "Exited (0) 2 weeks ago"), { s: "up", since: iso }).problem?.kind, "stopped", "it was running at the last look");
assert.equal(judge(info("x", "exited", "Exited (0) 2 weeks ago"), { s: "down", since: iso }).problem, null, "it was already down");
assert.equal(judge(info("x", "exited", "Exited (0) 2 days ago"), { s: "down", since: iso, alert: true }).problem?.kind, "stopped", "once raised it stays");
assert.equal(judge(info("x", "exited", "Exited (2) 20 hours ago")).problem?.text, "has stopped (exit code 2)");
assert.equal(judge(info("x", "dead", "Dead"), { s: "up", since: iso }).problem?.text, "is dead");
});
test("created and never started is said only after it has been so for an hour", () => {
const first = judge(info("kestra", "created", "Created"));
assert.equal(first.problem, null);
assert.deepEqual(first.next, { s: "created", since: new Date(NOW).toISOString() });
assert.equal(judge(info("kestra", "created", "Created"), { s: "created", since: new Date(NOW - 30 * 60_000).toISOString() }).problem, null);
assert.equal(judge(info("kestra", "created", "Created"), { s: "created", since: new Date(NOW - 2 * HOUR).toISOString() }).problem?.kind, "created");
});
test("what is remembered: running is up, and a recovered one is no longer alerting", () => {
assert.deepEqual(judge(info("x", "running", "Up 1 hour"), { s: "down", since: "2026-10-23T00:00:00.000Z", alert: true }).next, { s: "up", since: new Date(NOW).toISOString() });
assert.equal(judge(info("x", "paused", "Up 1 hour (Paused)")).problem, null);
});
});
describe("a whole Docker integration, over several looks", () => {
/** A few of the containers from a real export. */
const export1 = [
info("netbox-netbox-worker-1", "restarting", "Restarting (127) 39 seconds ago"),
info("portall-socket-proxy", "running", "Up 6 days (unhealthy)"),
info("drawio", "running", "Up 4 days (unhealthy)", "Siri", 2),
info("grocy", "exited", "Exited (0) 15 minutes ago", "Gabriella", 3),
info("planka-planka-1", "exited", "Exited (0) 2 weeks ago", "Siri", 2),
info("vikunja-vikunja-1", "exited", "Exited (2) 3 months ago", "Nova", 4),
info("nextcloud-aio-watchtower", "exited", "Exited (0) 2 days ago"),
info("snipe-itnew-app-1", "created", "Created", "Nova", 4),
info("dockhand-grype-1792611378733-7036sl", "created", "Created", "Anna", 5),
info("gitea", "running", "Up 4 days"),
];
const snap = (containers = export1, unreachable: import("../src/services/containerHealth.js").DockerSnapshot["unreachable"] = []): import("../src/services/containerHealth.js").DockerSnapshot => ({ integrationId: 7, integrationName: "Dockhand", containers, unreachable });
test("the first look finds what is wrong now, and nothing about what has long been stopped", () => {
const { conditions, next } = c.evaluateContainers([snap()], {}, NOW, []);
assert.deepEqual(conditions.map((x) => x.key).sort(), [
"container-restarting:7:1:netbox-netbox-worker-1",
"container-stopped:7:3:grocy",
"container-unhealthy:7:1:portall-socket-proxy",
"container-unhealthy:7:2:drawio",
]);
const worker = conditions.find((x) => x.key.startsWith("container-restarting"))!;
assert.deepEqual([worker.severity, worker.source, worker.message], ["critical", "dockhand:7:1", "netbox-netbox-worker-1 [Jackie] keeps restarting (exit code 127)"]);
assert.equal(conditions.find((x) => x.key.includes("grocy"))!.message, "grocy [Gabriella] has stopped");
assert.ok(next["7:4:vikunja-vikunja-1"] && !next["7:4:vikunja-vikunja-1"].alert, "remembered as down, quietly");
assert.equal(next["7:5:dockhand-grype-1792611378733-7036sl"], undefined, "the scanner is not remembered at all");
});
test("a stop that was raised stays raised, a container that starts again clears, and one that is left alone is not said", () => {
const first = c.evaluateContainers([snap()], {}, NOW, []);
const later = c.evaluateContainers([snap()], first.next, NOW + 2 * 24 * HOUR, []);
assert.ok(later.conditions.some((x) => x.key.includes("grocy")), "two days on, grocy is still stopped and still said");
assert.ok(!later.conditions.some((x) => x.key.includes("planka")), "planka never was");
const grocyBack = export1.map((x) => (x.name === "grocy" ? info("grocy", "running", "Up 3 minutes", "Gabriella", 3) : x));
const cleared = c.evaluateContainers([snap(grocyBack)], later.next, NOW + 3 * 24 * HOUR, []);
assert.ok(!cleared.conditions.some((x) => x.key.includes("grocy")));
const stoppedAgain = export1.map((x) => (x.name === "gitea" ? info("gitea", "exited", "Exited (1) 5 minutes ago") : x));
const caught = c.evaluateContainers([snap(stoppedAgain)], first.next, NOW + 15 * 60_000, []);
assert.equal(caught.conditions.find((x) => x.key.includes("gitea"))?.message, "gitea [Jackie] has stopped (exit code 1)", "running at the last look, stopped now");
const quiet = c.evaluateContainers([snap()], {}, NOW, ["Siri/drawio", "portall-*", "netbox-netbox-worker-1", "grocy"]);
assert.deepEqual(quiet.conditions, []);
});
test("a Docker host that cannot be read is said, and its containers are neither judged nor forgotten", () => {
const first = c.evaluateContainers([snap()], {}, NOW, []);
const down = c.evaluateContainers([snap(export1.filter((x) => x.environmentId !== 1), [{ environmentId: 1, environmentName: "Jackie", error: "connect ECONNREFUSED" }])], first.next, NOW + HOUR, []);
assert.ok(down.conditions.some((x) => x.key === "container-host:7:1" && x.message === "Dockhand: can't reach the Docker host Jackie — connect ECONNREFUSED"));
assert.ok(!down.conditions.some((x) => x.key === "container-restarting:7:1:netbox-netbox-worker-1"), "not judged while the host is out of reach (the alert is held by the caller)");
assert.ok(down.next["7:1:gitea"], "and what was remembered of Jackie's containers is kept");
const gone = c.evaluateContainers([snap(export1.filter((x) => x.name !== "gitea"))], first.next, NOW + HOUR, []);
assert.equal(gone.next["7:1:gitea"], undefined, "a container that has been removed is forgotten when its host is read");
});
});
describe("through the health pass, the Alerts page and the Docker page", () => {
let dockhand: Server;
let app: Server;
let base = "";
let who = "admin";
let integrationId = 0;
let annaUp = true;
let containers: Record<string, unknown>[] = [];
const row = (name: string, state: string, status: string) => ({ id: `id-${name}`, name, image: "img", state, status });
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
dockhand = createHttpServer((req, res) => {
const url = new URL(req.url ?? "", "http://x");
res.setHeader("content-type", "application/json");
if (url.pathname === "/api/environments") return void res.end(JSON.stringify([{ id: 1, name: "Jackie", connectionType: "hawser" }, { id: 2, name: "Anna", connectionType: "hawser" }]));
if (url.pathname === "/api/containers/pending-updates") return void res.end(JSON.stringify({ pendingUpdates: [] }));
if (url.pathname === "/api/containers") {
if (url.searchParams.get("env") === "2" && !annaUp) {
res.statusCode = 502;
return void res.end(JSON.stringify({ error: "agent not connected" }));
}
return void res.end(JSON.stringify(url.searchParams.get("env") === "1" ? containers : [row("ollama", "running", "Up 2 days")]));
}
res.statusCode = 404;
res.end("{}");
});
await new Promise<void>((r) => dockhand.listen(0, "127.0.0.1", r));
integrationId = (await db.insert(integrations).values({ type: "dockhand", name: "Dockhand", baseUrl: "", config: JSON.stringify({ url: `http://127.0.0.1:${(dockhand.address() as AddressInfo).port}`, token: "t" }) }).returning({ id: integrations.id }))[0].id;
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/integrations", integrationsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/integrations/${integrationId}/dockhand`;
});
after(() => {
dockhand.close();
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass reads Dockhand and finds the broken ones; the Alerts page says them under Docker, with a link", async () => {
containers = [row("netbox-worker", "restarting", "Restarting (127) 39 seconds ago"), row("gitea", "running", "Up 4 days"), row("drawio", "running", "Up 4 days (unhealthy)")];
const { snapshot } = await collectSnapshot();
assert.deepEqual(snapshot.docker?.map((d) => [d.integrationName, d.containers.length, d.unreachable.length]), [["Dockhand", 4, 0]]);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
assert.deepEqual(list.filter((x) => x.key.startsWith("container-")).map((x) => x.key).sort(), [`container-restarting:${integrationId}:1:netbox-worker`, `container-unhealthy:${integrationId}:1:drawio`]);
assert.equal(subjectOfConditionKey(`container-restarting:${integrationId}:1:netbox-worker`), `integration:${integrationId}`, "maintenance on the integration holds it");
invalidateAlertsCache();
const report = await getAlerts(true);
const a = report.alerts.find((x) => x.id === `services:container-restarting:${integrationId}:1:netbox-worker`)!;
assert.ok(a);
assert.deepEqual([a.severity, a.category, a.source, a.link], ["critical", "services", "Docker", "/docker"]);
});
test("a pass remembers what it saw, so a container that stops afterwards is caught; and an unreachable host is said and held", async () => {
await runHealthCheck(Date.now());
const memory = await c.loadContainerMemory();
assert.equal(memory[`${integrationId}:1:gitea`]?.s, "up");
containers = containers.map((x) => (x.name === "gitea" ? row("gitea", "exited", "Exited (1) 2 minutes ago") : x));
annaUp = false;
invalidateAlertsCache();
const report = await getAlerts(true);
const ids = report.alerts.map((x) => x.id);
assert.ok(ids.includes(`services:container-stopped:${integrationId}:1:gitea`), "gitea stopped after being seen running");
assert.ok(ids.includes(`services:container-host:${integrationId}:2`), "Anna cannot be reached");
assert.match(report.alerts.find((x) => x.id.endsWith(`container-host:${integrationId}:2`))!.message, /can't reach the Docker host Anna — agent not connected/);
annaUp = true;
});
test("the Docker page gets each container's problem and whether it is left alone, for anyone signed in", async () => {
who = "viewer";
const body = (await (await fetch(`${base}/containers`, { headers: { connection: "close" } })).json()) as any;
const gitea = body.containers.find((x: any) => x.name === "gitea");
assert.deepEqual([gitea.problem?.kind, gitea.leftAlone], ["stopped", false]);
assert.equal(body.containers.find((x: any) => x.name === "ollama").problem, null);
assert.equal(body.alerts, true);
who = "admin";
});
test("an administrator can leave a container alone and watch it again; nobody else can, and it is in the audit log", async () => {
const put = (leaveAlone: boolean) => fetch(`${base}/leave-alone`, { method: "PUT", headers: { "content-type": "application/json", connection: "close" }, body: JSON.stringify({ environmentName: "Jackie", name: "gitea", leaveAlone }) });
who = "viewer";
assert.equal((await put(true)).status, 403);
who = "admin";
assert.equal((await put(true)).status, 200);
assert.deepEqual((await getSettings()).healthChecks.containerIgnore, ["Jackie/gitea"]);
let body = (await (await fetch(`${base}/containers`, { headers: { connection: "close" } })).json()) as any;
assert.deepEqual([body.containers.find((x: any) => x.name === "gitea").leftAlone, body.containers.find((x: any) => x.name === "gitea").problem], [true, null]);
invalidateAlertsCache();
assert.ok(!(await getAlerts(true)).alerts.some((x) => x.id.includes("container-stopped") && x.id.endsWith("gitea")), "no alert for it now");
assert.equal((await put(false)).status, 200);
assert.deepEqual((await getSettings()).healthChecks.containerIgnore, []);
const audited = await db.select().from(auditLog).where(eq(auditLog.targetType, "container"));
assert.deepEqual(audited.map((a) => [a.action, a.targetId]), [["container_leave_alone", "Jackie/gitea"], ["container_watch", "Jackie/gitea"]]);
body = (await (await fetch(`${base}/containers`, { headers: { connection: "close" } })).json()) as any;
assert.equal(body.containers.find((x: any) => x.name === "gitea").leftAlone, false);
});
test("it can be switched off, and then nothing is asked of Dockhand", async () => {
const checks = (await getSettings()).healthChecks;
assert.equal(checks.containerAlerts, true, "on by default");
await updateSettings({ healthChecks: { ...checks, containerAlerts: false } });
assert.equal((await collectSnapshot()).snapshot.docker, undefined);
await updateSettings({ healthChecks: { ...checks, containerAlerts: true } });
});
});
+5 -3
View File
@@ -234,17 +234,19 @@ describe("rules, the Alerts page and the pages that show them", () => {
});
test("a pool on a steady climb is forecast; a flat one, a wobbly one and one already over the limit are not", async () => {
// This one is judged on the Alerts page and by the routes, which read the real clock (a pool not reported for two days is not believed), so its history is laid out from the real now.
const T = Date.now();
const [pve] = await db.select().from(integrations).where(eq(integrations.type, "proxmox"));
const make = async (key: string, label: string, valueAt: (day: number) => number, latest: number) => {
const [s] = await db.insert(storageSeries).values({ integrationId: pve.id, key, label, latestPercent: latest, usedBytes: null, totalBytes: null, updatedAt: new Date(NOW).toISOString() }).returning();
for (let d = 0; d <= 14; d += 0.5) await db.insert(storageSamples).values({ seriesId: s.id, at: new Date(NOW - (14 - d) * DAY).toISOString(), value: valueAt(d) });
const [s] = await db.insert(storageSeries).values({ integrationId: pve.id, key, label, latestPercent: latest, usedBytes: null, totalBytes: null, updatedAt: new Date(T).toISOString() }).returning();
for (let d = 0; d <= 14; d += 0.5) await db.insert(storageSamples).values({ seriesId: s.id, at: new Date(T - (14 - d) * DAY).toISOString(), value: valueAt(d) });
return s;
};
const climbing = await make("a:climb", "PVE / a: storage \"climb\"", (d) => 50 + d * 2.5, 85);
await make("b:flat", "PVE / b: storage \"flat\"", () => 40, 40);
await make("c:wobble", "PVE / c: storage \"wobble\"", (d) => 50 + (Math.floor(d * 2) % 2 === 0 ? 8 : -8), 58);
await make("d:over", "PVE / d: storage \"over\"", (d) => 80 + d * 0.9, 92.6);
const rows = await storageOverview(new Date(NOW));
const rows = await storageOverview(new Date(T));
const by = Object.fromEntries(rows.map((r) => [r.key, r]));
assert.equal(by["a:climb"].steady, true);
assert.equal(by["b:flat"].trend?.daysToFull ?? null, null);