Broken Docker containers are alerts

From Dockhand, on the health pass: a container that keeps restarting (critical, with
its exit code), one whose own health check fails, one that stopped unexpectedly, one
created and never started for an hour, and a Docker host Dockhand cannot reach. A stop
is only news when it was seen happen, so stacks switched off long ago stay quiet.
A Leave alone button on the Docker page and a list in Settings; on by default.

Also fixes a date-dependent storage-forecast test that pinned its clock while the code
under test reads the real one.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-24 21:03:56 +02:00
1 parent 16b24f5436
commit 0fa50c2d5a
20 files changed
+659 -17

No files matched your search

+24 -2
View File
@@ -17,6 +17,7 @@ import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createTailscaleAdapter, isKeyExpiringSoon } from "../integrations/tailscale/adapter.js";
import { createGiteaAdapter } from "../integrations/gitea/adapter.js";
import { createDockhandAdapter } from "../integrations/dockhand/adapter.js";
import { isLeftAlone, loadContainerMemory, memoryKey, problemsFor } from "../services/containerHealth.js";
import { createSemaphoreAdapter } from "../integrations/semaphore/adapter.js";
import { createProxmoxAdapter, guestsWithoutBackupCoverage } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter } from "../integrations/synology/adapter.js";
@@ -29,6 +30,7 @@ import { removeIntegrationData, syncUnifiIntegration } from "../services/unifiSy
import { removeHomeAssistantData, syncHomeAssistant } from "../services/homeAssistantSync.js";
import { forgetUps, syncUps } from "../services/upsSync.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { ID_PATTERNS, isValidId } from "../utils/pathSegment.js";
@@ -546,9 +548,15 @@ integrationsRouter.get("/:id/dockhand/containers", asyncHandler(async (req, res)
if (!found) return;
try {
const containers = await found.adapter.listContainers();
const read = await found.adapter.listContainersDetailed();
const containers = read.containers;
// What is wrong with each, as the health pass would say it (from what it remembers; nothing is changed here), and what is left alone.
const { healthChecks } = await getSettings();
const problems = problemsFor({ integrationId: found.integration.id, integrationName: found.integration.name, containers, unreachable: read.unreachable }, await loadContainerMemory(), Date.now(), healthChecks.containerIgnore);
res.json({
containers,
containers: containers.map((c) => ({ ...c, problem: problems[memoryKey(found.integration.id, c)] ?? null, leftAlone: isLeftAlone(c, healthChecks.containerIgnore) })),
unreachable: read.unreachable,
alerts: healthChecks.containerAlerts,
summary: {
total: containers.length,
running: containers.filter((c) => c.state === "running").length,
@@ -559,6 +567,20 @@ integrationsRouter.get("/:id/dockhand/containers", asyncHandler(async (req, res)
}
}));
// Leaves a container alone — no alert for it — or watches it again. Administrators: it changes what is alerted.
integrationsRouter.put("/:id/dockhand/leave-alone", requireRole("admin"), asyncHandler(async (req, res) => {
const body = z.object({ environmentName: z.string().min(1).max(100), name: z.string().min(1).max(100), leaveAlone: z.boolean() }).safeParse(req.body);
if (!body.success) return res.status(400).json({ error: "invalid_body" });
const entry = `${body.data.environmentName}/${body.data.name}`;
const { healthChecks } = await getSettings();
const list = healthChecks.containerIgnore.filter((p) => p !== entry && p !== body.data.name);
if (body.data.leaveAlone) list.push(entry);
await updateSettings({ healthChecks: { ...healthChecks, containerIgnore: list.slice(0, 200) } });
await recordAudit({ actor: req.currentUser!, category: "settings", action: body.data.leaveAlone ? "container_leave_alone" : "container_watch", targetType: "container", targetId: entry });
invalidateAlertsCache();
res.json({ containerIgnore: list });
}));
integrationsRouter.post(
"/:id/dockhand/check-updates",
requireRole("operator"),
+1 -1
View File
@@ -144,7 +144,7 @@ const updateSchema = z.object({
.partial()
.optional(),
healthChecks: z
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365), baselineAlerts: z.boolean() })
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365), baselineAlerts: z.boolean(), containerAlerts: z.boolean(), containerIgnore: z.array(z.string().trim().min(1).max(100)).max(200) })
.partial()
.optional(),
quietHours: z