Runbooks per alert kind, high-vulnerability threshold, wall display extras
A "What to do" note (text and/or link) can be written once for a whole kind of alert and shows beside every alert of that kind; alerts now carry their kind and source type. New optional setting: warn when a Docker host has N or more high vulnerabilities (off by default). The wall display gains tiles (Hetzner and game servers running, critical vulnerabilities, stuck downloads) and a summary line for each group of integrations, also in share links. Migration 0074; docs and tests included. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d9083adb5b
commit
3d9a2310bb
35 files changed
+9129
-30
No files matched your search
@@ -6,9 +6,16 @@ export type AlertSeverity = "critical" | "warning" | "info";
|
||||
/** What kind of problem — drives the filter on the Alerts page. */
|
||||
export type AlertCategory = "offline" | "disk" | "services" | "backup" | "updates" | "expiry" | "automation" | "integration" | "monitoring" | "tickets";
|
||||
|
||||
/** The kind of an alert: the first part of its key ("disk:server:3:/" is a "disk" alert, "hetzner-cert:2:20" a "hetzner-cert" one). What a kind-wide runbook is attached to. */
|
||||
export const kindOfKey = (key: string): string => key.split(":")[0] || key;
|
||||
|
||||
export interface Alert {
|
||||
/** Stable, so the page can key rows on it. */
|
||||
id: string;
|
||||
/** What kind of alert it is (see kindOfKey). */
|
||||
kind: string;
|
||||
/** The kind of source it is about — an integration's type ("hetzner", "sonarr"), "server", "slo" … — which the wall groups by. */
|
||||
type: string;
|
||||
severity: AlertSeverity;
|
||||
category: AlertCategory;
|
||||
/** Where it comes from, as a short name: "Server", "Proxmox", "Secrets", ... */
|
||||
|
||||
@@ -43,7 +43,7 @@ import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } fro
|
||||
import { endpointConditions } from "./endpointChecks.js";
|
||||
import { heartbeatConditions } from "./heartbeats.js";
|
||||
import { latestTests, restoreAlerts } from "./restoreTests.js";
|
||||
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
|
||||
import { kindOfKey, type Alert, type AlertCategory, type AlertSeverity, type SourceFailure } from "./alertTypes.js";
|
||||
|
||||
export interface AlertsReport {
|
||||
alerts: Alert[];
|
||||
@@ -125,7 +125,7 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "slo", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^docker-vuln:(\d+):/.exec(key);
|
||||
m = /^docker-vuln(?:-high)?:(\d+):/.exec(key);
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
|
||||
if (m) return { type: "speedtest", id: Number(m[1]), category: "monitoring" };
|
||||
@@ -198,6 +198,8 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const meta = SOURCES[a.type];
|
||||
alerts.push({
|
||||
id: `${a.category}:${a.key}`,
|
||||
kind: kindOfKey(a.key),
|
||||
type: a.type,
|
||||
severity: a.severity,
|
||||
category: a.category,
|
||||
source: meta?.label ?? sourceLabel(a.type),
|
||||
|
||||
@@ -120,10 +120,20 @@ export interface DockerVulnProblem {
|
||||
severity: "warning";
|
||||
}
|
||||
|
||||
/** Pure: a warning for each Docker host whose stored scans have critical findings. A host whose summary is unknown says nothing. */
|
||||
export function dockerVulnProblems(integrationId: number, rows: Pick<StateRow, "environmentId" | "environmentName" | "critical" | "high" | "findings">[]): DockerVulnProblem[] {
|
||||
/**
|
||||
* Pure: a warning for each Docker host whose stored scans have critical findings — and, when `highThreshold` is above 0, one for each whose scans have at least that many *high* ones. A host
|
||||
* whose summary is unknown says nothing.
|
||||
*/
|
||||
export function dockerVulnProblems(integrationId: number, rows: Pick<StateRow, "environmentId" | "environmentName" | "critical" | "high" | "findings">[], highThreshold = 0): DockerVulnProblem[] {
|
||||
const out: DockerVulnProblem[] = [];
|
||||
for (const r of [...rows].sort((a, b) => a.environmentName.localeCompare(b.environmentName))) {
|
||||
if (highThreshold > 0 && r.high !== null && r.high >= highThreshold) {
|
||||
out.push({
|
||||
key: `docker-vuln-high:${integrationId}:${r.environmentId}`,
|
||||
severity: "warning",
|
||||
message: `${r.environmentName}: ${r.high} high ${r.high === 1 ? "vulnerability" : "vulnerabilities"} in the scanned images (you asked to hear of ${highThreshold} or more)`,
|
||||
});
|
||||
}
|
||||
if (r.critical === null || r.critical <= 0) continue;
|
||||
const findings = findingsOf(r);
|
||||
const images = [...new Set(findings.map((f) => f.imageName))];
|
||||
|
||||
@@ -761,7 +761,7 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
held.add(`dockhand:${row.id}:vuln`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.dockerVulns ??= []).push({ integrationId: row.id, problems: dockerVulnProblems(row.id, stored) });
|
||||
(snapshot.dockerVulns ??= []).push({ integrationId: row.id, problems: dockerVulnProblems(row.id, stored, checks.vulnerabilityHighThreshold ?? 0) });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge the vulnerability scans:", err instanceof Error ? err.message : err);
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
/**
|
||||
* The kinds of integration in the groups the sidebar has, in its order. The same list as TYPE_GROUPS in web/src/utils/integrationTypes.ts — a test compares the two, so a kind added to one
|
||||
* and not the other is caught. A kind in no group (an older one) is simply not in any group's summary.
|
||||
*/
|
||||
export const INTEGRATION_GROUPS: { id: string; label: string; types: string[] }[] = [
|
||||
{ id: "infrastructure", label: "Infrastructure", types: ["proxmox", "synology", "dockhand", "hetzner", "fragnet", "tailscale"] },
|
||||
{ id: "apps", label: "Apps", types: ["homeassistant", "jellyfin", "sonarr", "radarr", "prowlarr", "immich", "paperless", "nextcloud", "authentik"] },
|
||||
{ id: "backups", label: "Backups", types: ["pbs"] },
|
||||
{ id: "network", label: "Network", types: ["unifi", "npm", "phpipam", "netbox"] },
|
||||
{ id: "monitoring", label: "Monitoring", types: ["uptimekuma", "prometheus", "ups", "speedtest"] },
|
||||
{ id: "automation", label: "Automation", types: ["semaphore", "gitea", "osticket", "n8n"] },
|
||||
];
|
||||
@@ -31,7 +31,7 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^docker-vuln:(\d+):/.exec(key);
|
||||
m = /^docker-vuln(?:-high)?:(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
|
||||
@@ -10,6 +10,7 @@ import { getAlerts } from "./alerts.js";
|
||||
import { getSourceStatus } from "./integrationHealthMonitor.js";
|
||||
import { describeTarget, listActiveWindows, activeSubjects } from "./maintenance.js";
|
||||
import { buildServerSummary } from "./serverSummary.js";
|
||||
import { buildWallExtras, type WallGroup, type WallTile } from "./wallExtras.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import type { ShareLinkDetail } from "../db/schema.js";
|
||||
|
||||
@@ -23,6 +24,9 @@ export interface PublicWall {
|
||||
/** Windows running now. */
|
||||
maintenance: number;
|
||||
integrationsFailing: number;
|
||||
/** What else is worth a glance (Hetzner servers running, critical vulnerabilities …) and a summary for each group of integrations — numbers and group names only, so every detail level has them. */
|
||||
tiles: WallTile[];
|
||||
groups: WallGroup[];
|
||||
/** Only with detail "names". */
|
||||
problems?: { id: string; severity: "critical" | "warning"; message: string }[];
|
||||
maintenanceWindows?: { id: number; name: string; endsAt: string }[];
|
||||
@@ -54,6 +58,8 @@ export async function buildPublicWall(detail: ShareLinkDetail, now: Date = new D
|
||||
const warning = report.counts.warning;
|
||||
const level: PublicWall["level"] = critical > 0 ? "critical" : warning > 0 ? "warning" : report.couldntCheck.length > 0 ? "unsure" : "ok";
|
||||
|
||||
const extras = await buildWallExtras(enabled.map((i) => ({ id: i.id, type: i.type })), new Set(failing.map((i) => i.type)), report.alerts, now.getTime());
|
||||
|
||||
const out: PublicWall = {
|
||||
detail,
|
||||
level,
|
||||
@@ -62,6 +68,8 @@ export async function buildPublicWall(detail: ShareLinkDetail, now: Date = new D
|
||||
alerts: { critical, warning },
|
||||
maintenance: windows.length,
|
||||
integrationsFailing: failing.length,
|
||||
tiles: extras.tiles,
|
||||
groups: extras.groups,
|
||||
generatedAt: now.toISOString(),
|
||||
display: { dateFormat: settings.display.dateFormat, timeFormat: settings.display.timeFormat },
|
||||
};
|
||||
|
||||
@@ -144,6 +144,8 @@ export interface HealthCheckSettings {
|
||||
containerAlerts: boolean;
|
||||
/** Whether critical vulnerabilities found by Dockhand's scans are alerted (services/dockerVulnSync.ts). */
|
||||
vulnerabilityAlerts: boolean;
|
||||
/** A Docker host whose scanned images have at least this many HIGH findings is a warning too (0 = off, the default; critical ones are alerted whatever this is, while vulnerabilityAlerts is on). */
|
||||
vulnerabilityHighThreshold: number;
|
||||
/** Containers to leave alone: a name, or "host/name"; * stands for anything. */
|
||||
containerIgnore: string[];
|
||||
}
|
||||
@@ -329,7 +331,7 @@ const DEFAULTS: AppSettings = {
|
||||
selfBackup: { enabled: true, time: "03:30", keep: 14 },
|
||||
profilePictures: { gravatar: false, identityProvider: false },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, vulnerabilityAlerts: true, containerIgnore: [] },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, vulnerabilityAlerts: true, vulnerabilityHighThreshold: 0, containerIgnore: [] },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
consistency: { excludedRanges: ["172.16.0.0/12"] },
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
/**
|
||||
* What the wall display shows beyond its four fixed tiles (servers online, alerts, in maintenance, integrations failing): a tile for each thing that is connected and worth a glance — Hetzner
|
||||
* servers running, game servers running, critical vulnerabilities, downloads that are stuck — and a summary for each group of integrations the menu has (Infrastructure, Apps, Network …): how
|
||||
* many are connected, how many are failing, how many alerts they have. Only numbers and the names of the groups, so a share link that gives "numbers only" may carry all of it.
|
||||
*
|
||||
* A tile is there only when its source is connected and was read lately: a number read an hour ago is not shown as how things are.
|
||||
*/
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { arrState, dockhandVulnerabilities, fragnetState, hetznerState, integrations } from "../db/schema.js";
|
||||
import type { Alert } from "./alertTypes.js";
|
||||
import { arrSyncStatuses, isArrFresh, storedOf as arrStoredOf, stuckItems } from "./arrSync.js";
|
||||
import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
|
||||
import { fragnetSyncStatuses, isFragnetFresh, serversOf as fragnetServersOf } from "./fragnetSync.js";
|
||||
import { hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf } from "./hetznerSync.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { getSourceStatus } from "./integrationHealthMonitor.js";
|
||||
import { INTEGRATION_GROUPS } from "./integrationGroups.js";
|
||||
|
||||
export type WallTileState = "ok" | "warn" | "bad" | "info";
|
||||
|
||||
export interface WallTile {
|
||||
id: string;
|
||||
label: string;
|
||||
figure: string;
|
||||
state: WallTileState;
|
||||
}
|
||||
|
||||
export interface WallGroup {
|
||||
id: string;
|
||||
label: string;
|
||||
/** Connected (and switched on). */
|
||||
integrations: number;
|
||||
failing: number;
|
||||
alerts: { critical: number; warning: number };
|
||||
}
|
||||
|
||||
export interface WallExtras {
|
||||
tiles: WallTile[];
|
||||
groups: WallGroup[];
|
||||
}
|
||||
|
||||
/** Pure: the groups, each with what is connected in it, what is failing and what is alerting — a group with nothing connected is left out. */
|
||||
export function buildGroups(connected: { id: number; type: string }[], failing: Set<string>, alerts: Pick<Alert, "type" | "severity" | "silenced">[]): WallGroup[] {
|
||||
const out: WallGroup[] = [];
|
||||
for (const g of INTEGRATION_GROUPS) {
|
||||
const mine = connected.filter((i) => g.types.includes(i.type));
|
||||
if (mine.length === 0) continue;
|
||||
const active = alerts.filter((a) => !a.silenced && g.types.includes(a.type));
|
||||
out.push({
|
||||
id: g.id,
|
||||
label: g.label,
|
||||
integrations: mine.length,
|
||||
failing: mine.filter((i) => failing.has(i.type)).length,
|
||||
alerts: { critical: active.filter((a) => a.severity === "critical").length, warning: active.filter((a) => a.severity === "warning").length },
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const plural = (n: number, one: string, many: string) => (n === 1 ? one : many);
|
||||
|
||||
/** Pure: how many of some servers run, as a tile — "3/3"; a stop is a warning for a cloud server that is billed, and only a note for a game server, which is often stopped on purpose. */
|
||||
export function runningTile(id: string, label: string, running: number, total: number, stopIs: "warn" | "info"): WallTile {
|
||||
return { id, label, figure: `${running}/${total}`, state: running === total ? "ok" : stopIs };
|
||||
}
|
||||
|
||||
export async function buildWallExtras(
|
||||
connected: { id: number; type: string }[],
|
||||
failing: Set<string>,
|
||||
alerts: Pick<Alert, "type" | "severity" | "silenced">[],
|
||||
nowMs: number = Date.now(),
|
||||
): Promise<WallExtras> {
|
||||
const tiles: WallTile[] = [];
|
||||
const ids = (types: string[]) => connected.filter((i) => types.includes(i.type)).map((i) => i.id);
|
||||
|
||||
try {
|
||||
const hz = ids(["hetzner"]);
|
||||
if (hz.length > 0) {
|
||||
const statuses = await hetznerSyncStatuses();
|
||||
let running = 0;
|
||||
let total = 0;
|
||||
let read = false;
|
||||
for (const row of await db.select().from(hetznerState)) {
|
||||
if (!hz.includes(row.integrationId) || !isHetznerFresh(statuses[row.integrationId], nowMs)) continue;
|
||||
read = true;
|
||||
for (const s of hetznerServersOf(row)) {
|
||||
total++;
|
||||
if (s.status === "running") running++;
|
||||
}
|
||||
}
|
||||
if (read && total > 0) tiles.push(runningTile("hetzner", "Hetzner servers running", running, total, "warn"));
|
||||
}
|
||||
|
||||
const fn = ids(["fragnet"]);
|
||||
if (fn.length > 0) {
|
||||
const statuses = await fragnetSyncStatuses();
|
||||
let running = 0;
|
||||
let total = 0;
|
||||
let read = false;
|
||||
for (const row of await db.select().from(fragnetState)) {
|
||||
if (!fn.includes(row.integrationId) || !isFragnetFresh(statuses[row.integrationId], nowMs)) continue;
|
||||
read = true;
|
||||
for (const s of fragnetServersOf(row)) {
|
||||
if (s.suspended) continue;
|
||||
total++;
|
||||
if (s.state === "running") running++;
|
||||
}
|
||||
}
|
||||
if (read && total > 0) tiles.push(runningTile("game-servers", "game servers running", running, total, "info"));
|
||||
}
|
||||
|
||||
const dh = ids(["dockhand"]);
|
||||
if (dh.length > 0) {
|
||||
const statuses = await dockerVulnSyncStatuses();
|
||||
let critical = 0;
|
||||
let known = false;
|
||||
for (const row of await db.select().from(dockhandVulnerabilities).where(inArray(dockhandVulnerabilities.integrationId, dh))) {
|
||||
if (!isDockerVulnFresh(statuses[row.integrationId], nowMs) || row.critical === null) continue;
|
||||
known = true;
|
||||
critical += row.critical;
|
||||
}
|
||||
if (known) tiles.push({ id: "vulnerabilities", label: `critical ${plural(critical, "vulnerability", "vulnerabilities")}`, figure: String(critical), state: critical > 0 ? "bad" : "ok" });
|
||||
}
|
||||
|
||||
const media = ids(["sonarr", "radarr"]);
|
||||
if (media.length > 0) {
|
||||
const statuses = await arrSyncStatuses();
|
||||
let stuck = 0;
|
||||
let known = false;
|
||||
for (const row of await db.select().from(arrState).where(inArray(arrState.integrationId, media))) {
|
||||
const state = arrStoredOf(row);
|
||||
if (!isArrFresh(statuses[row.integrationId], nowMs) || state.queue === null) continue;
|
||||
known = true;
|
||||
stuck += stuckItems(state.queue, nowMs).length;
|
||||
}
|
||||
if (known) tiles.push({ id: "stuck-downloads", label: `stuck ${plural(stuck, "download", "downloads")}`, figure: String(stuck), state: stuck > 0 ? "warn" : "ok" });
|
||||
}
|
||||
} catch (err) {
|
||||
// A tile that cannot be worked out is simply not there; the board must never fail for the sake of an extra.
|
||||
console.error("[wall] couldn't work out the extra tiles:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
return { tiles, groups: buildGroups(connected, failing, alerts) };
|
||||
}
|
||||
|
||||
/** The extras for the signed-in wall, which has none of it read yet: the connected integrations, which kinds are failing, and the alerts there are. */
|
||||
export async function wallExtrasNow(nowMs: number = Date.now()): Promise<WallExtras> {
|
||||
const [rows, report] = await Promise.all([db.select({ id: integrations.id, type: integrations.type }).from(integrations).where(and(eq(integrations.enabled, true))), getAlerts(false)]);
|
||||
const failing = new Set(rows.filter((r) => getSourceStatus(r.type).status === "failing").map((r) => r.type));
|
||||
return buildWallExtras(rows, failing, report.alerts, nowMs);
|
||||
}
|
||||
Reference in new issue
Block a user