Broken Docker containers are alerts

From Dockhand, on the health pass: a container that keeps restarting (critical, with
its exit code), one whose own health check fails, one that stopped unexpectedly, one
created and never started for an hour, and a Docker host Dockhand cannot reach. A stop
is only news when it was seen happen, so stacks switched off long ago stay quiet.
A Leave alone button on the Docker page and a list in Settings; on by default.

Also fixes a date-dependent storage-forecast test that pinned its clock while the code
under test reads the real one.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-24 21:03:56 +02:00
1 parent 16b24f5436
commit 0fa50c2d5a
20 files changed
+659 -17

No files matched your search

+14
View File
@@ -537,6 +537,10 @@ export interface HealthCheckSettings {
osSupportWarnDays: number;
/** A server whose CPU or memory over the last hour is clearly higher than is normal for this hour of the week is a warning. */
baselineAlerts: boolean;
/** A container that keeps restarting, is unhealthy, stopped unexpectedly or never started, and a Docker host that cannot be reached, is an alert. */
containerAlerts: boolean;
/** Containers to leave alone: a name, or "host/name"; * stands for anything. */
containerIgnore: string[];
}
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. */
@@ -2243,9 +2247,17 @@ export interface DockhandContainer {
updateAvailable: boolean | null;
newerVersion: string | null;
checkedAt: string | null;
/** What is wrong with it, as the health pass would say it; null when nothing is. */
problem?: { kind: "restarting" | "unhealthy" | "stopped" | "created"; text: string; severity: "critical" | "warning" } | null;
/** It is on the list of containers to leave alone, so nothing is alerted about it. */
leftAlone?: boolean;
}
export interface DockhandContainersResponse {
/** Docker hosts Dockhand could not read this time. */
unreachable?: { environmentId: number; environmentName: string; error: string }[];
/** Container alerts are switched on. */
alerts?: boolean;
containers: DockhandContainer[];
summary: { total: number; running: number };
}
@@ -3120,6 +3132,8 @@ export const api = {
dockhand: {
containers: (integrationId: number) =>
request<DockhandContainersResponse>(`/api/integrations/${integrationId}/dockhand/containers`),
leaveAlone: (integrationId: number, environmentName: string, name: string, leaveAlone: boolean) =>
request<{ containerIgnore: string[] }>(`/api/integrations/${integrationId}/dockhand/leave-alone`, { method: "PUT", body: JSON.stringify({ environmentName, name, leaveAlone }) }),
start: (integrationId: number, envId: number, containerId: string) =>
request<void>(
`/api/integrations/${integrationId}/dockhand/environments/${envId}/containers/${encodeURIComponent(containerId)}/start`,
+38 -1
View File
@@ -32,6 +32,17 @@ function updateBadge(c: DockhandContainer) {
);
}
/** What is wrong with a container, in a badge: the same finding the health pass raises an alert for. */
function problemBadge(c: DockhandContainer) {
if (c.leftAlone) return <span className="badge bg-secondary-lt text-secondary" title="No alert is raised for it">left alone</span>;
if (!c.problem) return null;
return (
<span className={`badge ${c.problem.severity === "critical" ? "bg-red-lt text-red" : "bg-yellow-lt text-yellow"}`} title={`An alert is raised: ${c.name} ${c.problem.text}`}>
{c.problem.kind === "restarting" ? "restart loop" : c.problem.kind === "unhealthy" ? "unhealthy" : c.problem.kind === "created" ? "never started" : "stopped unexpectedly"}
</span>
);
}
function containerStateBadge(state: string) {
switch (state) {
case "running":
@@ -95,6 +106,17 @@ export default function Docker({ user }: { user: CurrentUser }) {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [selectedId]);
async function leaveAlone(c: DockhandContainer) {
if (!selected) return;
try {
await api.integrations.dockhand.leaveAlone(selected.id, c.environmentName, c.name, !c.leftAlone);
toast.success(c.leftAlone ? `Watching ${c.name} again.` : `Leaving ${c.name} alone — no alerts for it.`);
loadContainers(selected.id);
} catch (err) {
toast.error(err instanceof Error ? err.message : String(err));
}
}
const hosts = [...new Set((data?.containers ?? []).map((c) => c.environmentName))].sort();
const filteredContainers = (data?.containers ?? []).filter((c) => !hostFilter || c.environmentName === hostFilter);
@@ -189,6 +211,13 @@ export default function Docker({ user }: { user: CurrentUser }) {
</div>
) : (
<div className="card">
{data?.unreachable && data.unreachable.length > 0 && (
<div className="alert alert-warning m-3 mb-0" role="status">
<div>
Dockhand can&rsquo;t reach {data.unreachable.map((u) => `${u.environmentName} (${u.error})`).join(", ")} — those hosts&rsquo; containers are missing from this list.
</div>
</div>
)}
<div className="card-header">
<h3 className="card-title">
Containers
@@ -197,6 +226,9 @@ export default function Docker({ user }: { user: CurrentUser }) {
{data.summary.running}/{data.summary.total} running
</span>
)}
{data && data.containers.filter((c) => c.problem && !c.leftAlone).length > 0 && (
<span className="text-red fw-normal ms-2">{data.containers.filter((c) => c.problem && !c.leftAlone).length} with a problem</span>
)}
{data && data.containers.filter((c) => c.updateAvailable).length > 0 && (
<span className="text-yellow fw-normal ms-2">
{data.containers.filter((c) => c.updateAvailable).length} update(s) available
@@ -270,7 +302,7 @@ export default function Docker({ user }: { user: CurrentUser }) {
<td className="text-secondary text-break">{c.environmentName}</td>
<td className="text-secondary text-break">{c.image}</td>
<td>
{containerStateBadge(c.state)}
{containerStateBadge(c.state)} {problemBadge(c)}
<div className="text-secondary small">{c.status}</div>
</td>
<td>{updateBadge(c)}</td>
@@ -303,6 +335,11 @@ export default function Docker({ user }: { user: CurrentUser }) {
Start
</button>
)}
{user.role === "admin" && (c.problem || c.leftAlone) && (
<button className="btn btn-sm btn-outline-secondary" onClick={() => void leaveAlone(c)} title={c.leftAlone ? "Raise alerts about it again" : "Stop raising alerts about this container"}>
{c.leftAlone ? "Watch again" : "Leave alone"}
</button>
)}
</div>
</td>
)}
@@ -84,7 +84,7 @@ const DIGEST_DAYS: { value: number; label: string }[] = [
{ value: 6, label: "Saturday" },
{ value: 0, label: "Sunday" },
];
const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true };
const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, containerIgnore: [] };
const DEFAULT_MONTHLY: MonthlyReportSettings = { enabled: false, quarterly: false, annual: false, day: 1, time: "08:00" };
type TestResult = { ok: boolean; message: string } | null;
@@ -841,6 +841,30 @@ export default function NotificationSettings() {
/>
{toggle("pbsVerificationCheck", "Proxmox Backup Server snapshot failed verification")}
{toggle("dockerUpdateCheck", "Docker image update available")}
<label className="form-check mt-3 mb-0">
<input type="checkbox" className="form-check-input" checked={healthChecks.containerAlerts} onChange={(e) => setHealthChecks((h) => ({ ...h, containerAlerts: e.target.checked }))} />
<span className="form-check-label">
Warn when a Docker container is broken
<span className="form-hint d-block">
From Dockhand, on the same pass as the server checks: a container that keeps restarting (critical), one whose own health check fails, one that stopped when it was not meant to
(it was running at the last look, or stopped within the day), one created and never started for an hour, and a Docker host Dockhand cannot reach. A container that has been stopped all along is
left alone.
</span>
</span>
</label>
<label className="form-label mt-2" htmlFor="container-ignore">
Containers to leave alone
</label>
<textarea
id="container-ignore"
className="form-control"
rows={3}
disabled={!healthChecks.containerAlerts}
placeholder={"one per line: a name, or host/name — * stands for anything\nplanka-*\nSiri/drawio"}
value={healthChecks.containerIgnore.join("\n")}
onChange={(e) => setHealthChecks((h) => ({ ...h, containerIgnore: e.target.value.split("\n").map((l) => l.trim()).filter(Boolean) }))}
/>
<div className="form-hint">The scanner containers Dockhand starts (dockhand-grype-*, dockhand-trivy-*) are always left alone. There is also a <em>Leave alone</em> button on the Docker page.</div>
</div>
</div>
</div>
+1
View File
@@ -23,6 +23,7 @@ export const SETTINGS_INDEX: SettingsEntry[] = [
{ title: "Away mode: hold notifications while I'm away", where: "Notifications › When", to: `${N}?tab=when`, words: "away holiday vacation travel back hold digest pause mute do not disturb critical" },
{ title: "Let critical alerts through quiet hours", where: "Notifications › When", to: `${N}?tab=when`, words: "quiet hours critical night wake sleep through urgent" },
{ title: "Quarterly and annual reports", where: "Notifications › When", to: `${N}?tab=when`, words: "report quarterly annual yearly quarter year send message monthly" },
{ title: "Warn when a Docker container is broken, and containers to leave alone", where: "Notifications › What to send › Backups and updates", to: `${N}?tab=events`, words: "docker container restarting restart loop unhealthy stopped exited created dockhand hawser ignore leave alone" },
{ title: "Warn when CPU or memory is unusually high for the time", where: "Notifications › What to send › Servers and disks", to: `${N}?tab=events`, words: "baseline usual normal unusual unusually busy high cpu memory load weekday hour history anomaly" },
{ title: "Warn when a server starts listening on a new port", where: "Notifications › What to send › Servers and disks", to: `${N}?tab=events`, words: "new port baseline listening tcp monitoring" },
{ title: "Call an alert flapping after (raises in an hour)", where: "Notifications › What to send › Servers and disks", to: `${N}?tab=events`, words: "flapping flap noise repeat raised cleared again hold back" },