Broken Docker containers are alerts
From Dockhand, on the health pass: a container that keeps restarting (critical, with its exit code), one whose own health check fails, one that stopped unexpectedly, one created and never started for an hour, and a Docker host Dockhand cannot reach. A stop is only news when it was seen happen, so stacks switched off long ago stay quiet. A Leave alone button on the Docker page and a list in Settings; on by default. Also fixes a date-dependent storage-forecast test that pinned its clock while the code under test reads the real one. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
16b24f5436
commit
0fa50c2d5a
20 files changed
+659
-17
No files matched your search
@@ -17,6 +17,7 @@ import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createTailscaleAdapter, isKeyExpiringSoon } from "../integrations/tailscale/adapter.js";
|
||||
import { createGiteaAdapter } from "../integrations/gitea/adapter.js";
|
||||
import { createDockhandAdapter } from "../integrations/dockhand/adapter.js";
|
||||
import { isLeftAlone, loadContainerMemory, memoryKey, problemsFor } from "../services/containerHealth.js";
|
||||
import { createSemaphoreAdapter } from "../integrations/semaphore/adapter.js";
|
||||
import { createProxmoxAdapter, guestsWithoutBackupCoverage } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter } from "../integrations/synology/adapter.js";
|
||||
@@ -29,6 +30,7 @@ import { removeIntegrationData, syncUnifiIntegration } from "../services/unifiSy
|
||||
import { removeHomeAssistantData, syncHomeAssistant } from "../services/homeAssistantSync.js";
|
||||
import { forgetUps, syncUps } from "../services/upsSync.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { ID_PATTERNS, isValidId } from "../utils/pathSegment.js";
|
||||
|
||||
@@ -546,9 +548,15 @@ integrationsRouter.get("/:id/dockhand/containers", asyncHandler(async (req, res)
|
||||
if (!found) return;
|
||||
|
||||
try {
|
||||
const containers = await found.adapter.listContainers();
|
||||
const read = await found.adapter.listContainersDetailed();
|
||||
const containers = read.containers;
|
||||
// What is wrong with each, as the health pass would say it (from what it remembers; nothing is changed here), and what is left alone.
|
||||
const { healthChecks } = await getSettings();
|
||||
const problems = problemsFor({ integrationId: found.integration.id, integrationName: found.integration.name, containers, unreachable: read.unreachable }, await loadContainerMemory(), Date.now(), healthChecks.containerIgnore);
|
||||
res.json({
|
||||
containers,
|
||||
containers: containers.map((c) => ({ ...c, problem: problems[memoryKey(found.integration.id, c)] ?? null, leftAlone: isLeftAlone(c, healthChecks.containerIgnore) })),
|
||||
unreachable: read.unreachable,
|
||||
alerts: healthChecks.containerAlerts,
|
||||
summary: {
|
||||
total: containers.length,
|
||||
running: containers.filter((c) => c.state === "running").length,
|
||||
@@ -559,6 +567,20 @@ integrationsRouter.get("/:id/dockhand/containers", asyncHandler(async (req, res)
|
||||
}
|
||||
}));
|
||||
|
||||
// Leaves a container alone — no alert for it — or watches it again. Administrators: it changes what is alerted.
|
||||
integrationsRouter.put("/:id/dockhand/leave-alone", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const body = z.object({ environmentName: z.string().min(1).max(100), name: z.string().min(1).max(100), leaveAlone: z.boolean() }).safeParse(req.body);
|
||||
if (!body.success) return res.status(400).json({ error: "invalid_body" });
|
||||
const entry = `${body.data.environmentName}/${body.data.name}`;
|
||||
const { healthChecks } = await getSettings();
|
||||
const list = healthChecks.containerIgnore.filter((p) => p !== entry && p !== body.data.name);
|
||||
if (body.data.leaveAlone) list.push(entry);
|
||||
await updateSettings({ healthChecks: { ...healthChecks, containerIgnore: list.slice(0, 200) } });
|
||||
await recordAudit({ actor: req.currentUser!, category: "settings", action: body.data.leaveAlone ? "container_leave_alone" : "container_watch", targetType: "container", targetId: entry });
|
||||
invalidateAlertsCache();
|
||||
res.json({ containerIgnore: list });
|
||||
}));
|
||||
|
||||
integrationsRouter.post(
|
||||
"/:id/dockhand/check-updates",
|
||||
requireRole("operator"),
|
||||
|
||||
@@ -144,7 +144,7 @@ const updateSchema = z.object({
|
||||
.partial()
|
||||
.optional(),
|
||||
healthChecks: z
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365), baselineAlerts: z.boolean() })
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365), baselineAlerts: z.boolean(), containerAlerts: z.boolean(), containerIgnore: z.array(z.string().trim().min(1).max(100)).max(200) })
|
||||
.partial()
|
||||
.optional(),
|
||||
quietHours: z
|
||||
|
||||
Reference in new issue
Block a user