Dockhand vulnerability scans: warn about critical findings, with a setting to turn it off

Reads what Dockhand's scans have already stored (never starts one) every
15 minutes and raises a warning per Docker host with critical findings.
Unknown counts and stale readings are held, not treated as clean.
Settings -> Notifications has a switch (on by default); the Docker page
shows counts, scan coverage and the critical findings; /metrics and docs
updated.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-25 22:34:06 +01:00
1 parent 65214f6f0f
commit 5a2b320175
25 files changed
+8527 -12

No files matched your search

+3 -1
View File
@@ -119,6 +119,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "slo", id: Number(m[1]), category: "monitoring" };
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
m = /^docker-vuln:(\d+):/.exec(key);
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
@@ -272,7 +274,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
}
}).finally(() => {
offlineKnown(new Set());
+149
View File
@@ -0,0 +1,149 @@
/**
* Keeps what Dockhand's vulnerability scans say up to date: every fifteen minutes each Dockhand integration is asked, for each of its Docker hosts, how many findings its stored scans
* add up to (and the critical ones, with the containers that use the image), and that is stored. Pages and alerts are built from what is stored, so none of them talks to Dockhand.
*
* Nothing here starts a scan: Dockhand scans when it is told to (by hand, after a pull, or for a whole host), and this only reads what it kept. So an image that was never scanned is
* not known to be clean — the page says how many of a host's images have a scan — and an environment whose summary could not be read is *unknown*, never *none*.
*
* What is raised is **critical** findings only (a host with one or more is a warning), and only while the setting for it is on (Settings → Notifications). Dockhand can also notify
* about vulnerabilities itself; if both are switched on you hear twice.
*/
import { and, eq, notInArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { dockhandVulnerabilities, integrations } from "../db/schema.js";
import { createDockhandAdapter, type DockhandEnvironmentVulns, type DockhandVulnFinding } from "../integrations/dockhand/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 15 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 45 * 60_000;
const STATUS_FLAG = "dockerVulnSync";
export interface DockerVulnSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, DockerVulnSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, DockerVulnSyncStatus>) : {};
} catch {
return {};
}
}
export async function dockerVulnSyncStatuses(): Promise<Record<number, DockerVulnSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isDockerVulnFresh = (status: DockerVulnSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof dockhandVulnerabilities.$inferSelect;
export const findingsOf = (row: Pick<StateRow, "findings">): DockhandVulnFinding[] => {
try {
const v = JSON.parse(row.findings);
return Array.isArray(v) ? (v as DockhandVulnFinding[]) : [];
} catch {
return [];
}
};
/** Writes down one read, replacing the last one for this integration (a host Dockhand no longer lists is dropped). */
export async function storeRead(integrationId: number, environments: DockhandEnvironmentVulns[], now: Date = new Date()): Promise<void> {
for (const e of environments) {
const values = {
environmentName: e.environmentName,
critical: e.summary?.critical ?? null,
high: e.summary?.high ?? null,
medium: e.summary?.medium ?? null,
low: e.summary?.low ?? null,
imagesScanned: e.summary?.imagesScanned ?? null,
totalImages: e.summary?.totalImages ?? null,
findings: JSON.stringify(e.critical),
updatedAt: now.toISOString(),
};
await db.insert(dockhandVulnerabilities).values({ integrationId, environmentId: e.environmentId, ...values }).onConflictDoUpdate({ target: [dockhandVulnerabilities.integrationId, dockhandVulnerabilities.environmentId], set: values });
}
const ids = environments.map((e) => e.environmentId);
await db.delete(dockhandVulnerabilities).where(ids.length > 0 ? and(eq(dockhandVulnerabilities.integrationId, integrationId), notInArray(dockhandVulnerabilities.environmentId, ids)) : eq(dockhandVulnerabilities.integrationId, integrationId));
}
/** Reads one Dockhand's scans and stores them. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncDockerVulns(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<{ environments: DockhandEnvironmentVulns[] }>): Promise<DockerVulnSyncStatus> {
let status: DockerVulnSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "dockhand") throw new Error("not a Dockhand integration");
const result = await (read ? read(loaded.config) : createDockhandAdapter(loaded.config as never).readVulnerabilities());
await storeRead(integrationId, result.environments, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[docker-vulns] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[docker-vulns] couldn't record the result:", err);
}
return status;
}
export async function syncAllDockerVulns(now: Date = new Date()): Promise<Record<number, DockerVulnSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "dockhand"), eq(integrations.enabled, true)));
const out: Record<number, DockerVulnSyncStatus> = {};
for (const r of rows) out[r.id] = await syncDockerVulns(r.id, now);
return out;
}
export async function forgetDockerVulns(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface DockerVulnProblem {
key: string;
message: string;
severity: "warning";
}
/** Pure: a warning for each Docker host whose stored scans have critical findings. A host whose summary is unknown says nothing. */
export function dockerVulnProblems(integrationId: number, rows: Pick<StateRow, "environmentId" | "environmentName" | "critical" | "high" | "findings">[]): DockerVulnProblem[] {
const out: DockerVulnProblem[] = [];
for (const r of [...rows].sort((a, b) => a.environmentName.localeCompare(b.environmentName))) {
if (r.critical === null || r.critical <= 0) continue;
const findings = findingsOf(r);
const images = [...new Set(findings.map((f) => f.imageName))];
const fixable = findings.filter((f) => f.fixedVersion).length;
const where = images.length === 0 ? "" : ` in ${images.length === 1 ? "1 image" : `${images.length}${findings.length >= r.critical ? "" : "+"} images`}: ${images.slice(0, 3).join(", ")}${images.length > 3 ? ` and ${images.length - 3} more` : ""}`;
out.push({
key: `docker-vuln:${integrationId}:${r.environmentId}`,
severity: "warning",
message: `${r.environmentName}: ${r.critical} critical ${r.critical === 1 ? "vulnerability" : "vulnerabilities"}${where}${findings.length > 0 ? ` (${fixable} of the first ${findings.length} ${fixable === 1 ? "has" : "have"} a fix)` : ""}`,
});
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initDockerVulnSync(): void {
if (timer) return;
const pass = () => void syncAllDockerVulns().catch((err) => console.error("[docker-vulns] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 27_000);
console.log(`[docker-vulns] reading Dockhand's vulnerability scans every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+24 -1
View File
@@ -1,6 +1,6 @@
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
@@ -28,6 +28,7 @@ import { isPaperlessFresh, paperlessProblems, paperlessSyncStatuses, type Paperl
import { isN8nFresh, n8nProblems, n8nSyncStatuses, type N8nProblem } from "./n8nSync.js";
import { isNetBoxFresh, netboxProblems, netboxSyncStatuses, type NetBoxProblem } from "./netboxSync.js";
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type NextcloudProblem } from "./nextcloudSync.js";
import { dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, type DockerVulnProblem } from "./dockerVulnSync.js";
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
import { recordStorageSamples } from "./storageHistory.js";
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
@@ -116,6 +117,8 @@ export interface HealthSnapshot {
/** Each Docker integration's containers as last read, with what was remembered of them (services/containerHealth.ts). */
docker?: DockerSnapshot[];
dockerMemory?: ContainerMemory;
/** Critical vulnerabilities in the images of each Dockhand's Docker hosts, from its last scans, only while that reading is recent (services/dockerVulnSync.ts). */
dockerVulns?: { integrationId: number; problems: DockerVulnProblem[] }[];
/** Containers to leave alone. */
dockerIgnore?: string[];
/** The clock the container judgements are made on (the pass's, so a test can set it). */
@@ -254,6 +257,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
for (const c of evaluateContainers(snapshot.docker, snapshot.dockerMemory ?? {}, snapshot.dockerNow ?? now, snapshot.dockerIgnore ?? []).conditions) out.push(c);
}
// Critical vulnerabilities in the images Dockhand has scanned.
for (const v of snapshot.dockerVulns ?? []) for (const p of v.problems) out.push({ key: p.key, source: `dockhand:${v.integrationId}:vuln`, message: p.message, severity: p.severity });
// A service-level target that has used most of its month's error budget, or all of it.
for (const f of snapshot.levels ?? []) out.push({ key: levelKey(f), source: "slo", message: f.message, severity: f.severity });
@@ -646,6 +652,23 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
}
}
// Vulnerabilities Dockhand's scans found, as the sync pass stored them (only critical ones are raised). Switched off with the setting; a reading gone stale is held, not cleared.
if (checks.vulnerabilityAlerts) {
try {
const statuses = await dockerVulnSyncStatuses();
for (const row of await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "dockhand"), eq(integrations.enabled, true)))) {
const stored = await db.select().from(dockhandVulnerabilities).where(eq(dockhandVulnerabilities.integrationId, row.id));
if (stored.length === 0 || !isDockerVulnFresh(statuses[row.id], Date.now())) {
held.add(`dockhand:${row.id}:vuln`);
continue;
}
(snapshot.dockerVulns ??= []).push({ integrationId: row.id, problems: dockerVulnProblems(row.id, stored) });
}
} catch (err) {
console.error("[health] couldn't judge the vulnerability scans:", err instanceof Error ? err.message : err);
}
}
// Service-level targets: how much of the month's error budget is used. A failure here costs only this check.
try {
snapshot.levels = await levelFindings();
+2
View File
@@ -31,6 +31,8 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `server:${m[1]}`;
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^docker-vuln:(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
+19 -1
View File
@@ -5,12 +5,13 @@
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState } from "../db/schema.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { backupState, listAll as listDuplicati } from "./duplicati.js";
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
import { isNextcloudFresh, nextcloudSyncStatuses } from "./nextcloudSync.js";
import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
import { isN8nFresh, n8nSyncStatuses, workflowsOf as n8nWorkflowsOf } from "./n8nSync.js";
import { componentsOf as paperlessComponentsOf, failedOf as paperlessFailedOf, isPaperlessFresh, paperlessSyncStatuses } from "./paperlessSync.js";
@@ -83,6 +84,22 @@ async function addNextcloudMetrics(out: Exposition, now: Date) {
}
}
/** Each Docker host's vulnerability counts from Dockhand's scans, for /metrics. Only while the reading is recent, and only for hosts whose summary could be read. */
async function addDockerVulnMetrics(out: Exposition, now: Date) {
const statuses = await dockerVulnSyncStatuses();
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "dockhand"))).map((r) => [r.id, r]));
for (const s of await db.select().from(dockhandVulnerabilities)) {
const row = names.get(s.integrationId);
if (!row || !row.enabled || s.critical === null || !isDockerVulnFresh(statuses[s.integrationId], now.getTime())) continue;
const labels = { integration: row.name, host: s.environmentName };
out.add("homelab_docker_vulnerabilities", "gauge", "Vulnerabilities in the scanned images of a Docker host, by severity, from Dockhand's stored scans.", s.critical, { ...labels, severity: "critical" });
if (s.high !== null) out.add("homelab_docker_vulnerabilities", "gauge", "Vulnerabilities in the scanned images of a Docker host, by severity, from Dockhand's stored scans.", s.high, { ...labels, severity: "high" });
if (s.medium !== null) out.add("homelab_docker_vulnerabilities", "gauge", "Vulnerabilities in the scanned images of a Docker host, by severity, from Dockhand's stored scans.", s.medium, { ...labels, severity: "medium" });
if (s.low !== null) out.add("homelab_docker_vulnerabilities", "gauge", "Vulnerabilities in the scanned images of a Docker host, by severity, from Dockhand's stored scans.", s.low, { ...labels, severity: "low" });
if (s.imagesScanned !== null) out.add("homelab_docker_images_scanned", "gauge", "How many of a Docker host's images have a vulnerability scan in Dockhand.", s.imagesScanned, labels);
}
}
/** Each NetBox, for /metrics: the workers running and what it holds. Only while the reading is recent. */
async function addNetBoxMetrics(out: Exposition, now: Date) {
const statuses = await netboxSyncStatuses();
@@ -388,6 +405,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addN8nMetrics(out, now);
await addNetBoxMetrics(out, now);
await addNextcloudMetrics(out, now);
await addDockerVulnMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}
+3 -1
View File
@@ -142,6 +142,8 @@ export interface HealthCheckSettings {
baselineAlerts: boolean;
/** A container that keeps restarting, is unhealthy, stopped unexpectedly or never started, and a Docker host Dockhand cannot reach, is an alert (services/containerHealth.ts). */
containerAlerts: boolean;
/** Whether critical vulnerabilities found by Dockhand's scans are alerted (services/dockerVulnSync.ts). */
vulnerabilityAlerts: boolean;
/** Containers to leave alone: a name, or "host/name"; * stands for anything. */
containerIgnore: string[];
}
@@ -327,7 +329,7 @@ const DEFAULTS: AppSettings = {
selfBackup: { enabled: true, time: "03:30", keep: 14 },
profilePictures: { gravatar: false, identityProvider: false },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, containerIgnore: [] },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, vulnerabilityAlerts: true, containerIgnore: [] },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
consistency: { excludedRanges: ["172.16.0.0/12"] },