Broken Docker containers are alerts
From Dockhand, on the health pass: a container that keeps restarting (critical, with its exit code), one whose own health check fails, one that stopped unexpectedly, one created and never started for an hour, and a Docker host Dockhand cannot reach. A stop is only news when it was seen happen, so stacks switched off long ago stay quiet. A Leave alone button on the Docker page and a list in Settings; on by default. Also fixes a date-dependent storage-forecast test that pinned its clock while the code under test reads the real one. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
16b24f5436
commit
0fa50c2d5a
20 files changed
+659
-17
No files matched your search
@@ -108,6 +108,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "ups", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^slo:(\d+)$/.exec(key);
|
||||
if (m) return { type: "slo", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
return { type: "server", id: null, category: "disk" };
|
||||
}
|
||||
|
||||
@@ -230,7 +232,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
@@ -239,7 +241,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Containers that are broken, said before somebody notices the app behind them is gone: one that keeps restarting, one whose own health check says it is unhealthy, one that
|
||||
* stopped without anyone meaning it to, one that was created and never started, and a Docker host Dockhand can no longer reach. Read from Dockhand (the Docker integration) on
|
||||
* the health pass; nothing is sent to Docker.
|
||||
*
|
||||
* The hard part is *stopped*. Plenty of containers are stopped on purpose — a stack that is switched off, a one-shot job, a watchtower that ran and exited — and nobody wants an
|
||||
* alert for each of them for ever. So a stop is only news when it was seen happen: the container was running at the previous look, or (for one the app has never seen) it
|
||||
* stopped within the last day, as Docker's own status text says. One that has been down all along is left alone, and so is anything named in the list of containers to leave alone.
|
||||
* An alert, once raised, stays until the container runs again, is removed, or is left alone.
|
||||
*/
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export interface ContainerInfo {
|
||||
environmentId: number;
|
||||
environmentName: string;
|
||||
name: string;
|
||||
state: string;
|
||||
status: string;
|
||||
}
|
||||
|
||||
export interface DockerSnapshot {
|
||||
integrationId: number;
|
||||
integrationName: string;
|
||||
containers: ContainerInfo[];
|
||||
/** Docker hosts Dockhand knows but could not read this time. */
|
||||
unreachable: { environmentId: number; environmentName: string; error: string }[];
|
||||
}
|
||||
|
||||
/** What was seen of each container, by "integration:environment:name", kept between looks. */
|
||||
export type ContainerMemory = Record<string, { s: "up" | "down" | "created"; since: string; alert?: boolean }>;
|
||||
|
||||
export interface ContainerCondition {
|
||||
key: string;
|
||||
source: string;
|
||||
message: string;
|
||||
severity: "critical" | "warning";
|
||||
}
|
||||
|
||||
/** A container created and never started is left a while (a stack is being brought up) before it is said. */
|
||||
export const CREATED_GRACE_MS = 60 * 60_000;
|
||||
/** A container the app has never seen running counts as "just stopped" if Docker says it stopped within this long. */
|
||||
export const RECENT_STOP_MS = 24 * 3_600_000;
|
||||
/** Containers Dockhand starts to scan images with, which come and go. Always left alone. */
|
||||
export const BUILT_IN_IGNORE = ["dockhand-grype-*", "dockhand-trivy-*"];
|
||||
|
||||
const UNITS: Record<string, number> = { second: 1_000, minute: 60_000, hour: 3_600_000, day: 86_400_000, week: 604_800_000, month: 2_592_000_000, year: 31_536_000_000 };
|
||||
|
||||
export interface ParsedStatus {
|
||||
unhealthy: boolean;
|
||||
/** The exit code in "Exited (127)" or "Restarting (127)". */
|
||||
exitCode: number | null;
|
||||
/** How long ago it exited, from "… 15 minutes ago"; null when the text does not say. */
|
||||
agoMs: number | null;
|
||||
}
|
||||
|
||||
/** Pure: what Docker's human status text says ("Up 6 days (unhealthy)", "Exited (0) 15 minutes ago", "Restarting (127) 39 seconds ago"). */
|
||||
export function parseStatus(status: string): ParsedStatus {
|
||||
const code = /^(?:Exited|Restarting)\s*\((\d+)\)/i.exec(status);
|
||||
const ago = /(?:(\d+)|an?|about an?|less than a)\s+(second|minute|hour|day|week|month|year)s?\s+ago/i.exec(status);
|
||||
return {
|
||||
unhealthy: /\(unhealthy\)/i.test(status),
|
||||
exitCode: code ? Number(code[1]) : null,
|
||||
agoMs: ago ? (ago[1] ? Number(ago[1]) : 1) * UNITS[ago[2].toLowerCase()] : null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Pure: whether a container is named in the list of ones to leave alone — by name, or "host/name"; * stands for anything. */
|
||||
export function isLeftAlone(c: Pick<ContainerInfo, "name" | "environmentName">, patterns: string[]): boolean {
|
||||
const match = (pattern: string, text: string) => new RegExp(`^${pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*")}$`, "i").test(text);
|
||||
return [...BUILT_IN_IGNORE, ...patterns].some((p) => match(p.trim(), c.name) || match(p.trim(), `${c.environmentName}/${c.name}`));
|
||||
}
|
||||
|
||||
export const memoryKey = (integrationId: number, c: Pick<ContainerInfo, "environmentId" | "name">) => `${integrationId}:${c.environmentId}:${c.name}`;
|
||||
|
||||
export type ProblemKind = "restarting" | "unhealthy" | "stopped" | "created";
|
||||
|
||||
export interface ContainerProblem {
|
||||
kind: ProblemKind;
|
||||
text: string;
|
||||
severity: "critical" | "warning";
|
||||
}
|
||||
|
||||
/** Pure: what is wrong with one container, given what was seen of it before (or nothing). Also what to remember of it. */
|
||||
export function judgeContainer(c: ContainerInfo, prev: ContainerMemory[string] | undefined, nowMs: number): { problem: ContainerProblem | null; next: ContainerMemory[string] } {
|
||||
const nowIso = new Date(nowMs).toISOString();
|
||||
const st = parseStatus(c.status);
|
||||
if (c.state === "restarting" || /^Restarting/i.test(c.status)) {
|
||||
return { problem: { kind: "restarting", text: `keeps restarting${st.exitCode !== null ? ` (exit code ${st.exitCode})` : ""}`, severity: "critical" }, next: { s: "up", since: prev?.since ?? nowIso } };
|
||||
}
|
||||
if (c.state === "running") {
|
||||
return { problem: st.unhealthy ? { kind: "unhealthy", text: "is unhealthy — its own health check is failing", severity: "warning" } : null, next: { s: "up", since: prev?.s === "up" ? prev.since : nowIso } };
|
||||
}
|
||||
if (c.state === "created") {
|
||||
const since = prev?.s === "created" ? prev.since : nowIso;
|
||||
const long = nowMs - Date.parse(since) >= CREATED_GRACE_MS;
|
||||
return { problem: long ? { kind: "created", text: "was created but has never been started", severity: "warning" } : null, next: { s: "created", since } };
|
||||
}
|
||||
if (c.state === "exited" || c.state === "dead") {
|
||||
const wasUp = prev?.s === "up";
|
||||
const justStopped = !prev && st.agoMs !== null && st.agoMs < RECENT_STOP_MS;
|
||||
const alert = prev?.alert === true || wasUp || justStopped;
|
||||
const text = c.state === "dead" ? "is dead" : `has stopped${st.exitCode ? ` (exit code ${st.exitCode})` : ""}`;
|
||||
return { problem: alert ? { kind: "stopped", text, severity: "warning" } : null, next: { s: "down", since: prev?.s === "down" ? prev.since : nowIso, ...(alert ? { alert: true } : {}) } };
|
||||
}
|
||||
// paused and anything else: not judged
|
||||
return { problem: null, next: { s: "down", since: prev?.s === "down" ? prev.since : nowIso } };
|
||||
}
|
||||
|
||||
/** Pure: the conditions for what was read, and what to remember. Reads `memory`, changes nothing. */
|
||||
export function evaluateContainers(snapshots: DockerSnapshot[], memory: ContainerMemory, nowMs: number, ignore: string[]): { conditions: ContainerCondition[]; next: ContainerMemory } {
|
||||
const conditions: ContainerCondition[] = [];
|
||||
const next: ContainerMemory = { ...memory };
|
||||
for (const snap of snapshots) {
|
||||
const unreachable = new Set(snap.unreachable.map((u) => u.environmentId));
|
||||
// What is remembered of the hosts that were read is rebuilt from what was read; a host that could not be read keeps what it had.
|
||||
for (const k of Object.keys(next)) {
|
||||
const [int, env] = k.split(":");
|
||||
if (Number(int) === snap.integrationId && !unreachable.has(Number(env))) delete next[k];
|
||||
}
|
||||
for (const u of snap.unreachable) {
|
||||
conditions.push({ key: `container-host:${snap.integrationId}:${u.environmentId}`, source: `dockhand:${snap.integrationId}`, severity: "warning", message: `${snap.integrationName}: can't reach the Docker host ${u.environmentName} — ${u.error}` });
|
||||
}
|
||||
for (const c of snap.containers) {
|
||||
if (unreachable.has(c.environmentId) || isLeftAlone(c, ignore)) continue;
|
||||
const key = memoryKey(snap.integrationId, c);
|
||||
const { problem, next: remembered } = judgeContainer(c, memory[key], nowMs);
|
||||
next[key] = remembered;
|
||||
if (!problem) continue;
|
||||
conditions.push({ key: `container-${problem.kind}:${snap.integrationId}:${c.environmentId}:${c.name}`, source: `dockhand:${snap.integrationId}:${c.environmentId}`, severity: problem.severity, message: `${c.name} [${c.environmentName}] ${problem.text}` });
|
||||
}
|
||||
}
|
||||
return { conditions, next };
|
||||
}
|
||||
|
||||
const MEMORY_FLAG = "containerMemory";
|
||||
|
||||
export async function loadContainerMemory(): Promise<ContainerMemory> {
|
||||
try {
|
||||
const raw = await getInternalFlag(MEMORY_FLAG);
|
||||
return raw ? (JSON.parse(raw) as ContainerMemory) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/** Keeps what was seen, for what is still wanted (integrations that no longer exist are dropped). */
|
||||
export async function saveContainerMemory(next: ContainerMemory, integrationIds: number[]): Promise<void> {
|
||||
const wanted = new Set(integrationIds);
|
||||
const kept: ContainerMemory = {};
|
||||
for (const [k, v] of Object.entries(next)) if (wanted.has(Number(k.split(":")[0]))) kept[k] = v;
|
||||
await setInternalFlag(MEMORY_FLAG, JSON.stringify(kept));
|
||||
}
|
||||
|
||||
/** What is wrong with each container, by memory key, for the Docker page (reads, never changes what is remembered). */
|
||||
export function problemsFor(snap: DockerSnapshot, memory: ContainerMemory, nowMs: number, ignore: string[]): Record<string, ContainerProblem> {
|
||||
const out: Record<string, ContainerProblem> = {};
|
||||
for (const c of snap.containers) {
|
||||
if (isLeftAlone(c, ignore)) continue;
|
||||
const { problem } = judgeContainer(c, memory[memoryKey(snap.integrationId, c)], nowMs);
|
||||
if (problem) out[memoryKey(snap.integrationId, c)] = problem;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
@@ -18,6 +18,8 @@ import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThres
|
||||
import { computeRuleFindings, findingKey, listRules, type RuleFinding } from "./alertRules.js";
|
||||
import { baselineKey, computeBaselineFindings, type BaselineFinding } from "./baselines.js";
|
||||
import { levelFindings, levelKey, type LevelFinding } from "./serviceLevels.js";
|
||||
import { createDockhandAdapter } from "../integrations/dockhand/adapter.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
|
||||
|
||||
@@ -84,6 +86,13 @@ export interface HealthSnapshot {
|
||||
baselines?: BaselineFinding[];
|
||||
/** Service-level targets whose error budget for the month is mostly or wholly used (services/serviceLevels.ts). */
|
||||
levels?: LevelFinding[];
|
||||
/** Each Docker integration's containers as last read, with what was remembered of them (services/containerHealth.ts). */
|
||||
docker?: DockerSnapshot[];
|
||||
dockerMemory?: ContainerMemory;
|
||||
/** Containers to leave alone. */
|
||||
dockerIgnore?: string[];
|
||||
/** The clock the container judgements are made on (the pass's, so a test can set it). */
|
||||
dockerNow?: number;
|
||||
}
|
||||
export interface Thresholds {
|
||||
serverOfflineMinutes: number;
|
||||
@@ -186,6 +195,11 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
}
|
||||
}
|
||||
|
||||
// Containers that are broken, stopped unexpectedly or never started, and Docker hosts that cannot be reached.
|
||||
if (snapshot.docker && snapshot.docker.length > 0) {
|
||||
for (const c of evaluateContainers(snapshot.docker, snapshot.dockerMemory ?? {}, snapshot.dockerNow ?? now, snapshot.dockerIgnore ?? []).conditions) out.push(c);
|
||||
}
|
||||
|
||||
// A service-level target that has used most of its month's error budget, or all of it.
|
||||
for (const f of snapshot.levels ?? []) out.push({ key: levelKey(f), source: "slo", message: f.message, severity: f.severity });
|
||||
|
||||
@@ -408,6 +422,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
}
|
||||
}
|
||||
|
||||
// Docker containers, from Dockhand: what is broken, what stopped unexpectedly. A Dockhand that cannot be read holds its containers' alerts rather than clearing them.
|
||||
if (checks.containerAlerts) {
|
||||
snapshot.dockerIgnore = checks.containerIgnore;
|
||||
snapshot.dockerMemory = await loadContainerMemory();
|
||||
for (const row of await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "dockhand"), eq(integrations.enabled, true)))) {
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(row.id);
|
||||
if (!loaded) continue;
|
||||
const read = await createDockhandAdapter(loaded.config as any).listContainersDetailed();
|
||||
(snapshot.docker ??= []).push({ integrationId: row.id, integrationName: row.name, containers: read.containers, unreachable: read.unreachable });
|
||||
for (const u of read.unreachable) held.add(`dockhand:${row.id}:${u.environmentId}`);
|
||||
} catch (err) {
|
||||
console.error(`[health] couldn't read Docker integration ${row.id}:`, err instanceof Error ? err.message : err);
|
||||
held.add(`dockhand:${row.id}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Service-level targets: how much of the month's error budget is used. A failure here costs only this check.
|
||||
try {
|
||||
snapshot.levels = await levelFindings();
|
||||
@@ -550,6 +582,16 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added:
|
||||
const { added: announced, resolved, next, newlyDown, ended } = diffConditions(await loadState(), current, held, await activeSubjects(new Date(now)), coveredKeys);
|
||||
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
|
||||
|
||||
// What was seen of the containers is kept for the next look (only this pass writes it; the Alerts page only reads it).
|
||||
if (snapshot.docker) {
|
||||
try {
|
||||
const after = evaluateContainers(snapshot.docker, snapshot.dockerMemory ?? {}, now, snapshot.dockerIgnore ?? []);
|
||||
await saveContainerMemory(after.next, (await db.select({ id: integrations.id }).from(integrations).where(eq(integrations.type, "dockhand"))).map((r) => r.id));
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't keep what was seen of the containers:", err);
|
||||
}
|
||||
}
|
||||
|
||||
// A host's message says what else it takes with it.
|
||||
const added = announced.map((c) => {
|
||||
const m = /^offline:(server|device):(\d+)$/.exec(c.key);
|
||||
|
||||
@@ -29,6 +29,8 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^baseline:(?:cpu|mem):server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^offline:device:(\d+)$/.exec(key);
|
||||
|
||||
@@ -140,6 +140,10 @@ export interface HealthCheckSettings {
|
||||
osSupportWarnDays: number;
|
||||
/** A server whose CPU load or memory use over the last hour is clearly higher than is normal for this hour of the week is a warning (services/baselines.ts). */
|
||||
baselineAlerts: boolean;
|
||||
/** A container that keeps restarting, is unhealthy, stopped unexpectedly or never started, and a Docker host Dockhand cannot reach, is an alert (services/containerHealth.ts). */
|
||||
containerAlerts: boolean;
|
||||
/** Containers to leave alone: a name, or "host/name"; * stands for anything. */
|
||||
containerIgnore: string[];
|
||||
}
|
||||
|
||||
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
|
||||
@@ -323,7 +327,7 @@ const DEFAULTS: AppSettings = {
|
||||
selfBackup: { enabled: true, time: "03:30", keep: 14 },
|
||||
profilePictures: { gravatar: false, identityProvider: false },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, containerIgnore: [] },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
consistency: { excludedRanges: ["172.16.0.0/12"] },
|
||||
|
||||
Reference in new issue
Block a user