Runbooks per alert kind, high-vulnerability threshold, wall display extras
A "What to do" note (text and/or link) can be written once for a whole kind of alert and shows beside every alert of that kind; alerts now carry their kind and source type. New optional setting: warn when a Docker host has N or more high vulnerabilities (off by default). The wall display gains tiles (Hetzner and game servers running, critical vulnerabilities, stuck downloads) and a summary line for each group of integrations, also in share links. Migration 0074; docs and tests included. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d9083adb5b
commit
3d9a2310bb
35 files changed
+9129
-30
No files matched your search
@@ -0,0 +1,8 @@
|
||||
CREATE TABLE `alert_kind_runbooks` (
|
||||
`kind` text PRIMARY KEY NOT NULL,
|
||||
`title` text NOT NULL,
|
||||
`body` text DEFAULT '' NOT NULL,
|
||||
`url` text,
|
||||
`updated_by` text,
|
||||
`updated_at` text NOT NULL
|
||||
);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -519,6 +519,13 @@
|
||||
"when": 1793126767536,
|
||||
"tag": "0073_chilly_silver_surfer",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 74,
|
||||
"version": "6",
|
||||
"when": 1793128598267,
|
||||
"tag": "0074_open_the_hood",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -70,6 +70,7 @@ import { hetznerRouter } from "./routes/hetzner.js";
|
||||
import { speedtestRouter } from "./routes/speedtest.js";
|
||||
import { fragnetRouter } from "./routes/fragnet.js";
|
||||
import { arrRouter } from "./routes/arr.js";
|
||||
import { wallRouter } from "./routes/wall.js";
|
||||
import { costsRouter } from "./routes/costs.js";
|
||||
import { nextcloudRouter } from "./routes/nextcloud.js";
|
||||
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
|
||||
@@ -218,6 +219,7 @@ export function createApp() {
|
||||
app.use("/api/speedtest", speedtestRouter);
|
||||
app.use("/api/fragnet", fragnetRouter);
|
||||
app.use("/api/arr", arrRouter);
|
||||
app.use("/api/wall", wallRouter);
|
||||
app.use("/api/costs", costsRouter);
|
||||
app.use("/api/nextcloud", nextcloudRouter);
|
||||
app.use("/api/service-levels", serviceLevelsRouter);
|
||||
|
||||
@@ -884,6 +884,18 @@ export const runbooks = sqliteTable(
|
||||
(t) => [index("runbooks_target_idx").on(t.targetType, t.targetId)],
|
||||
);
|
||||
|
||||
// What to do first about a KIND of alert, whichever server or integration it is about — "when a Hetzner certificate is about to end: renew it in the console, then check the
|
||||
// load balancer". The kind is the first part of an alert's key ("disk", "hetzner-cert", "sonarr-queue"). Shown beside every alert of that kind on the Alerts page, as a note and/or a
|
||||
// link. Read by routes/runbooks.ts.
|
||||
export const alertKindRunbooks = sqliteTable("alert_kind_runbooks", {
|
||||
kind: text("kind").primaryKey(),
|
||||
title: text("title").notNull(),
|
||||
body: text("body").notNull().default(""),
|
||||
url: text("url"), // a page to open (the host's documentation, a wiki page); null when there is none
|
||||
updatedBy: text("updated_by"),
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── Acknowledgements and incidents ─────────────────────────────────────────
|
||||
|
||||
// "I'm on it" for one alert — see services/alertAcks.ts. Keyed by the alert's stable id (not a foreign key to anything); gone once the alert has been.
|
||||
|
||||
@@ -605,6 +605,7 @@ integrationsRouter.get("/:id/dockhand/vulnerabilities", asyncHandler(async (req,
|
||||
const status = statuses[id] ?? null;
|
||||
res.json({
|
||||
alerts: settings.healthChecks.vulnerabilityAlerts,
|
||||
highThreshold: settings.healthChecks.vulnerabilityHighThreshold ?? 0,
|
||||
status,
|
||||
fresh: isDockerVulnFresh(status ?? undefined, Date.now()),
|
||||
environments: rows
|
||||
|
||||
@@ -2,7 +2,7 @@ import { Router } from "express";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { dnsProviders, integrations, maintenanceTargetTypes, networkDevices, runbooks, servers } from "../db/schema.js";
|
||||
import { alertKindRunbooks, dnsProviders, integrations, maintenanceTargetTypes, networkDevices, runbooks, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { describeTarget } from "../services/maintenance.js";
|
||||
@@ -25,6 +25,22 @@ const bodySchema = z.object({
|
||||
});
|
||||
const createSchema = bodySchema.extend({ targetType: z.enum(maintenanceTargetTypes), targetId: z.number().int().positive() });
|
||||
|
||||
/** What an alert kind looks like: the first part of a key, letters, digits and dashes. */
|
||||
const KIND = /^[A-Za-z0-9][A-Za-z0-9_-]{0,59}$/;
|
||||
const kindSchema = z.object({
|
||||
title: z.string().trim().min(1).max(120),
|
||||
body: z.string().max(20_000).default(""),
|
||||
/** A page to open; http or https only. */
|
||||
url: z
|
||||
.string()
|
||||
.trim()
|
||||
.max(500)
|
||||
.nullable()
|
||||
.optional()
|
||||
.transform((v) => (v === undefined || v === null || v === "" ? null : v))
|
||||
.refine((v) => v === null || /^https?:\/\/[^\s]+$/i.test(v), "A web address starting with http:// or https://"),
|
||||
});
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
// What there is for one thing: the full text.
|
||||
@@ -55,6 +71,35 @@ runbooksRouter.get("/summary", asyncHandler(async (_req, res) => {
|
||||
res.json({ runbooks: rows.filter((r) => exists[r.targetType].has(r.targetId)) });
|
||||
}));
|
||||
|
||||
// What there is for kinds of alert (by what they are about, wherever it happens), for everyone: it is what to do first, and is read by the Alerts page.
|
||||
runbooksRouter.get("/kinds", asyncHandler(async (_req, res) => {
|
||||
res.json({ kinds: await db.select().from(alertKindRunbooks).orderBy(alertKindRunbooks.kind) });
|
||||
}));
|
||||
|
||||
runbooksRouter.put("/kinds/:kind", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const kind = String(req.params.kind);
|
||||
const parsed = kindSchema.safeParse(req.body);
|
||||
if (!KIND.test(kind)) return res.status(400).json({ error: "invalid_kind" });
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a title, and a web address starting with http:// or https:// if you add one.", details: parsed.error.flatten() });
|
||||
if (parsed.data.body.trim() === "" && parsed.data.url === null) return res.status(400).json({ error: "invalid_body", message: "Write what to do, or give a page to open." });
|
||||
const actor = req.currentUser!;
|
||||
const [before] = await db.select({ kind: alertKindRunbooks.kind }).from(alertKindRunbooks).where(eq(alertKindRunbooks.kind, kind)).limit(1);
|
||||
const values = { title: parsed.data.title, body: parsed.data.body, url: parsed.data.url, updatedBy: actor.name ?? actor.email ?? actor.oidcSub, updatedAt: new Date().toISOString() };
|
||||
const [row] = await db.insert(alertKindRunbooks).values({ kind, ...values }).onConflictDoUpdate({ target: alertKindRunbooks.kind, set: values }).returning();
|
||||
await recordAudit({ actor, category: "runbook", action: before ? "update" : "create", targetType: "alert_kind", targetId: kind, detail: { title: parsed.data.title } });
|
||||
res.json({ runbook: row });
|
||||
}));
|
||||
|
||||
runbooksRouter.delete("/kinds/:kind", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const kind = String(req.params.kind);
|
||||
if (!KIND.test(kind)) return res.status(400).json({ error: "invalid_kind" });
|
||||
const [before] = await db.select().from(alertKindRunbooks).where(eq(alertKindRunbooks.kind, kind)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(alertKindRunbooks).where(eq(alertKindRunbooks.kind, kind));
|
||||
await recordAudit({ actor: req.currentUser!, category: "runbook", action: "delete", targetType: "alert_kind", targetId: kind, detail: { title: before.title } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
runbooksRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = createSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a title and some text, and say what it is for.", details: parsed.error.flatten() });
|
||||
|
||||
@@ -144,7 +144,7 @@ const updateSchema = z.object({
|
||||
.partial()
|
||||
.optional(),
|
||||
healthChecks: z
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365), baselineAlerts: z.boolean(), containerAlerts: z.boolean(), vulnerabilityAlerts: z.boolean().default(true), containerIgnore: z.array(z.string().trim().min(1).max(100)).max(200) })
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365), baselineAlerts: z.boolean(), containerAlerts: z.boolean(), vulnerabilityAlerts: z.boolean().default(true), vulnerabilityHighThreshold: z.number().int().min(0).max(10000).default(0), containerIgnore: z.array(z.string().trim().min(1).max(100)).max(200) })
|
||||
.partial()
|
||||
.optional(),
|
||||
quietHours: z
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { wallExtrasNow } from "../services/wallExtras.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const wallRouter = Router();
|
||||
wallRouter.use(requireAuth);
|
||||
|
||||
/** What the signed-in wall display shows besides the four tiles it works out itself: the extra tiles (Hetzner, game servers, vulnerabilities, stuck downloads) and a summary for each group of integrations. */
|
||||
wallRouter.get("/extras", asyncHandler(async (_req, res) => {
|
||||
res.json(await wallExtrasNow());
|
||||
}));
|
||||
@@ -6,9 +6,16 @@ export type AlertSeverity = "critical" | "warning" | "info";
|
||||
/** What kind of problem — drives the filter on the Alerts page. */
|
||||
export type AlertCategory = "offline" | "disk" | "services" | "backup" | "updates" | "expiry" | "automation" | "integration" | "monitoring" | "tickets";
|
||||
|
||||
/** The kind of an alert: the first part of its key ("disk:server:3:/" is a "disk" alert, "hetzner-cert:2:20" a "hetzner-cert" one). What a kind-wide runbook is attached to. */
|
||||
export const kindOfKey = (key: string): string => key.split(":")[0] || key;
|
||||
|
||||
export interface Alert {
|
||||
/** Stable, so the page can key rows on it. */
|
||||
id: string;
|
||||
/** What kind of alert it is (see kindOfKey). */
|
||||
kind: string;
|
||||
/** The kind of source it is about — an integration's type ("hetzner", "sonarr"), "server", "slo" … — which the wall groups by. */
|
||||
type: string;
|
||||
severity: AlertSeverity;
|
||||
category: AlertCategory;
|
||||
/** Where it comes from, as a short name: "Server", "Proxmox", "Secrets", ... */
|
||||
|
||||
@@ -43,7 +43,7 @@ import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } fro
|
||||
import { endpointConditions } from "./endpointChecks.js";
|
||||
import { heartbeatConditions } from "./heartbeats.js";
|
||||
import { latestTests, restoreAlerts } from "./restoreTests.js";
|
||||
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
|
||||
import { kindOfKey, type Alert, type AlertCategory, type AlertSeverity, type SourceFailure } from "./alertTypes.js";
|
||||
|
||||
export interface AlertsReport {
|
||||
alerts: Alert[];
|
||||
@@ -125,7 +125,7 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "slo", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^docker-vuln:(\d+):/.exec(key);
|
||||
m = /^docker-vuln(?:-high)?:(\d+):/.exec(key);
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
|
||||
if (m) return { type: "speedtest", id: Number(m[1]), category: "monitoring" };
|
||||
@@ -198,6 +198,8 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const meta = SOURCES[a.type];
|
||||
alerts.push({
|
||||
id: `${a.category}:${a.key}`,
|
||||
kind: kindOfKey(a.key),
|
||||
type: a.type,
|
||||
severity: a.severity,
|
||||
category: a.category,
|
||||
source: meta?.label ?? sourceLabel(a.type),
|
||||
|
||||
@@ -120,10 +120,20 @@ export interface DockerVulnProblem {
|
||||
severity: "warning";
|
||||
}
|
||||
|
||||
/** Pure: a warning for each Docker host whose stored scans have critical findings. A host whose summary is unknown says nothing. */
|
||||
export function dockerVulnProblems(integrationId: number, rows: Pick<StateRow, "environmentId" | "environmentName" | "critical" | "high" | "findings">[]): DockerVulnProblem[] {
|
||||
/**
|
||||
* Pure: a warning for each Docker host whose stored scans have critical findings — and, when `highThreshold` is above 0, one for each whose scans have at least that many *high* ones. A host
|
||||
* whose summary is unknown says nothing.
|
||||
*/
|
||||
export function dockerVulnProblems(integrationId: number, rows: Pick<StateRow, "environmentId" | "environmentName" | "critical" | "high" | "findings">[], highThreshold = 0): DockerVulnProblem[] {
|
||||
const out: DockerVulnProblem[] = [];
|
||||
for (const r of [...rows].sort((a, b) => a.environmentName.localeCompare(b.environmentName))) {
|
||||
if (highThreshold > 0 && r.high !== null && r.high >= highThreshold) {
|
||||
out.push({
|
||||
key: `docker-vuln-high:${integrationId}:${r.environmentId}`,
|
||||
severity: "warning",
|
||||
message: `${r.environmentName}: ${r.high} high ${r.high === 1 ? "vulnerability" : "vulnerabilities"} in the scanned images (you asked to hear of ${highThreshold} or more)`,
|
||||
});
|
||||
}
|
||||
if (r.critical === null || r.critical <= 0) continue;
|
||||
const findings = findingsOf(r);
|
||||
const images = [...new Set(findings.map((f) => f.imageName))];
|
||||
|
||||
@@ -761,7 +761,7 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
held.add(`dockhand:${row.id}:vuln`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.dockerVulns ??= []).push({ integrationId: row.id, problems: dockerVulnProblems(row.id, stored) });
|
||||
(snapshot.dockerVulns ??= []).push({ integrationId: row.id, problems: dockerVulnProblems(row.id, stored, checks.vulnerabilityHighThreshold ?? 0) });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge the vulnerability scans:", err instanceof Error ? err.message : err);
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
/**
|
||||
* The kinds of integration in the groups the sidebar has, in its order. The same list as TYPE_GROUPS in web/src/utils/integrationTypes.ts — a test compares the two, so a kind added to one
|
||||
* and not the other is caught. A kind in no group (an older one) is simply not in any group's summary.
|
||||
*/
|
||||
export const INTEGRATION_GROUPS: { id: string; label: string; types: string[] }[] = [
|
||||
{ id: "infrastructure", label: "Infrastructure", types: ["proxmox", "synology", "dockhand", "hetzner", "fragnet", "tailscale"] },
|
||||
{ id: "apps", label: "Apps", types: ["homeassistant", "jellyfin", "sonarr", "radarr", "prowlarr", "immich", "paperless", "nextcloud", "authentik"] },
|
||||
{ id: "backups", label: "Backups", types: ["pbs"] },
|
||||
{ id: "network", label: "Network", types: ["unifi", "npm", "phpipam", "netbox"] },
|
||||
{ id: "monitoring", label: "Monitoring", types: ["uptimekuma", "prometheus", "ups", "speedtest"] },
|
||||
{ id: "automation", label: "Automation", types: ["semaphore", "gitea", "osticket", "n8n"] },
|
||||
];
|
||||
@@ -31,7 +31,7 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^docker-vuln:(\d+):/.exec(key);
|
||||
m = /^docker-vuln(?:-high)?:(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
|
||||
@@ -10,6 +10,7 @@ import { getAlerts } from "./alerts.js";
|
||||
import { getSourceStatus } from "./integrationHealthMonitor.js";
|
||||
import { describeTarget, listActiveWindows, activeSubjects } from "./maintenance.js";
|
||||
import { buildServerSummary } from "./serverSummary.js";
|
||||
import { buildWallExtras, type WallGroup, type WallTile } from "./wallExtras.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import type { ShareLinkDetail } from "../db/schema.js";
|
||||
|
||||
@@ -23,6 +24,9 @@ export interface PublicWall {
|
||||
/** Windows running now. */
|
||||
maintenance: number;
|
||||
integrationsFailing: number;
|
||||
/** What else is worth a glance (Hetzner servers running, critical vulnerabilities …) and a summary for each group of integrations — numbers and group names only, so every detail level has them. */
|
||||
tiles: WallTile[];
|
||||
groups: WallGroup[];
|
||||
/** Only with detail "names". */
|
||||
problems?: { id: string; severity: "critical" | "warning"; message: string }[];
|
||||
maintenanceWindows?: { id: number; name: string; endsAt: string }[];
|
||||
@@ -54,6 +58,8 @@ export async function buildPublicWall(detail: ShareLinkDetail, now: Date = new D
|
||||
const warning = report.counts.warning;
|
||||
const level: PublicWall["level"] = critical > 0 ? "critical" : warning > 0 ? "warning" : report.couldntCheck.length > 0 ? "unsure" : "ok";
|
||||
|
||||
const extras = await buildWallExtras(enabled.map((i) => ({ id: i.id, type: i.type })), new Set(failing.map((i) => i.type)), report.alerts, now.getTime());
|
||||
|
||||
const out: PublicWall = {
|
||||
detail,
|
||||
level,
|
||||
@@ -62,6 +68,8 @@ export async function buildPublicWall(detail: ShareLinkDetail, now: Date = new D
|
||||
alerts: { critical, warning },
|
||||
maintenance: windows.length,
|
||||
integrationsFailing: failing.length,
|
||||
tiles: extras.tiles,
|
||||
groups: extras.groups,
|
||||
generatedAt: now.toISOString(),
|
||||
display: { dateFormat: settings.display.dateFormat, timeFormat: settings.display.timeFormat },
|
||||
};
|
||||
|
||||
@@ -144,6 +144,8 @@ export interface HealthCheckSettings {
|
||||
containerAlerts: boolean;
|
||||
/** Whether critical vulnerabilities found by Dockhand's scans are alerted (services/dockerVulnSync.ts). */
|
||||
vulnerabilityAlerts: boolean;
|
||||
/** A Docker host whose scanned images have at least this many HIGH findings is a warning too (0 = off, the default; critical ones are alerted whatever this is, while vulnerabilityAlerts is on). */
|
||||
vulnerabilityHighThreshold: number;
|
||||
/** Containers to leave alone: a name, or "host/name"; * stands for anything. */
|
||||
containerIgnore: string[];
|
||||
}
|
||||
@@ -329,7 +331,7 @@ const DEFAULTS: AppSettings = {
|
||||
selfBackup: { enabled: true, time: "03:30", keep: 14 },
|
||||
profilePictures: { gravatar: false, identityProvider: false },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, vulnerabilityAlerts: true, containerIgnore: [] },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, vulnerabilityAlerts: true, vulnerabilityHighThreshold: 0, containerIgnore: [] },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
consistency: { excludedRanges: ["172.16.0.0/12"] },
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
/**
|
||||
* What the wall display shows beyond its four fixed tiles (servers online, alerts, in maintenance, integrations failing): a tile for each thing that is connected and worth a glance — Hetzner
|
||||
* servers running, game servers running, critical vulnerabilities, downloads that are stuck — and a summary for each group of integrations the menu has (Infrastructure, Apps, Network …): how
|
||||
* many are connected, how many are failing, how many alerts they have. Only numbers and the names of the groups, so a share link that gives "numbers only" may carry all of it.
|
||||
*
|
||||
* A tile is there only when its source is connected and was read lately: a number read an hour ago is not shown as how things are.
|
||||
*/
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { arrState, dockhandVulnerabilities, fragnetState, hetznerState, integrations } from "../db/schema.js";
|
||||
import type { Alert } from "./alertTypes.js";
|
||||
import { arrSyncStatuses, isArrFresh, storedOf as arrStoredOf, stuckItems } from "./arrSync.js";
|
||||
import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
|
||||
import { fragnetSyncStatuses, isFragnetFresh, serversOf as fragnetServersOf } from "./fragnetSync.js";
|
||||
import { hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf } from "./hetznerSync.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { getSourceStatus } from "./integrationHealthMonitor.js";
|
||||
import { INTEGRATION_GROUPS } from "./integrationGroups.js";
|
||||
|
||||
export type WallTileState = "ok" | "warn" | "bad" | "info";
|
||||
|
||||
export interface WallTile {
|
||||
id: string;
|
||||
label: string;
|
||||
figure: string;
|
||||
state: WallTileState;
|
||||
}
|
||||
|
||||
export interface WallGroup {
|
||||
id: string;
|
||||
label: string;
|
||||
/** Connected (and switched on). */
|
||||
integrations: number;
|
||||
failing: number;
|
||||
alerts: { critical: number; warning: number };
|
||||
}
|
||||
|
||||
export interface WallExtras {
|
||||
tiles: WallTile[];
|
||||
groups: WallGroup[];
|
||||
}
|
||||
|
||||
/** Pure: the groups, each with what is connected in it, what is failing and what is alerting — a group with nothing connected is left out. */
|
||||
export function buildGroups(connected: { id: number; type: string }[], failing: Set<string>, alerts: Pick<Alert, "type" | "severity" | "silenced">[]): WallGroup[] {
|
||||
const out: WallGroup[] = [];
|
||||
for (const g of INTEGRATION_GROUPS) {
|
||||
const mine = connected.filter((i) => g.types.includes(i.type));
|
||||
if (mine.length === 0) continue;
|
||||
const active = alerts.filter((a) => !a.silenced && g.types.includes(a.type));
|
||||
out.push({
|
||||
id: g.id,
|
||||
label: g.label,
|
||||
integrations: mine.length,
|
||||
failing: mine.filter((i) => failing.has(i.type)).length,
|
||||
alerts: { critical: active.filter((a) => a.severity === "critical").length, warning: active.filter((a) => a.severity === "warning").length },
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const plural = (n: number, one: string, many: string) => (n === 1 ? one : many);
|
||||
|
||||
/** Pure: how many of some servers run, as a tile — "3/3"; a stop is a warning for a cloud server that is billed, and only a note for a game server, which is often stopped on purpose. */
|
||||
export function runningTile(id: string, label: string, running: number, total: number, stopIs: "warn" | "info"): WallTile {
|
||||
return { id, label, figure: `${running}/${total}`, state: running === total ? "ok" : stopIs };
|
||||
}
|
||||
|
||||
export async function buildWallExtras(
|
||||
connected: { id: number; type: string }[],
|
||||
failing: Set<string>,
|
||||
alerts: Pick<Alert, "type" | "severity" | "silenced">[],
|
||||
nowMs: number = Date.now(),
|
||||
): Promise<WallExtras> {
|
||||
const tiles: WallTile[] = [];
|
||||
const ids = (types: string[]) => connected.filter((i) => types.includes(i.type)).map((i) => i.id);
|
||||
|
||||
try {
|
||||
const hz = ids(["hetzner"]);
|
||||
if (hz.length > 0) {
|
||||
const statuses = await hetznerSyncStatuses();
|
||||
let running = 0;
|
||||
let total = 0;
|
||||
let read = false;
|
||||
for (const row of await db.select().from(hetznerState)) {
|
||||
if (!hz.includes(row.integrationId) || !isHetznerFresh(statuses[row.integrationId], nowMs)) continue;
|
||||
read = true;
|
||||
for (const s of hetznerServersOf(row)) {
|
||||
total++;
|
||||
if (s.status === "running") running++;
|
||||
}
|
||||
}
|
||||
if (read && total > 0) tiles.push(runningTile("hetzner", "Hetzner servers running", running, total, "warn"));
|
||||
}
|
||||
|
||||
const fn = ids(["fragnet"]);
|
||||
if (fn.length > 0) {
|
||||
const statuses = await fragnetSyncStatuses();
|
||||
let running = 0;
|
||||
let total = 0;
|
||||
let read = false;
|
||||
for (const row of await db.select().from(fragnetState)) {
|
||||
if (!fn.includes(row.integrationId) || !isFragnetFresh(statuses[row.integrationId], nowMs)) continue;
|
||||
read = true;
|
||||
for (const s of fragnetServersOf(row)) {
|
||||
if (s.suspended) continue;
|
||||
total++;
|
||||
if (s.state === "running") running++;
|
||||
}
|
||||
}
|
||||
if (read && total > 0) tiles.push(runningTile("game-servers", "game servers running", running, total, "info"));
|
||||
}
|
||||
|
||||
const dh = ids(["dockhand"]);
|
||||
if (dh.length > 0) {
|
||||
const statuses = await dockerVulnSyncStatuses();
|
||||
let critical = 0;
|
||||
let known = false;
|
||||
for (const row of await db.select().from(dockhandVulnerabilities).where(inArray(dockhandVulnerabilities.integrationId, dh))) {
|
||||
if (!isDockerVulnFresh(statuses[row.integrationId], nowMs) || row.critical === null) continue;
|
||||
known = true;
|
||||
critical += row.critical;
|
||||
}
|
||||
if (known) tiles.push({ id: "vulnerabilities", label: `critical ${plural(critical, "vulnerability", "vulnerabilities")}`, figure: String(critical), state: critical > 0 ? "bad" : "ok" });
|
||||
}
|
||||
|
||||
const media = ids(["sonarr", "radarr"]);
|
||||
if (media.length > 0) {
|
||||
const statuses = await arrSyncStatuses();
|
||||
let stuck = 0;
|
||||
let known = false;
|
||||
for (const row of await db.select().from(arrState).where(inArray(arrState.integrationId, media))) {
|
||||
const state = arrStoredOf(row);
|
||||
if (!isArrFresh(statuses[row.integrationId], nowMs) || state.queue === null) continue;
|
||||
known = true;
|
||||
stuck += stuckItems(state.queue, nowMs).length;
|
||||
}
|
||||
if (known) tiles.push({ id: "stuck-downloads", label: `stuck ${plural(stuck, "download", "downloads")}`, figure: String(stuck), state: stuck > 0 ? "warn" : "ok" });
|
||||
}
|
||||
} catch (err) {
|
||||
// A tile that cannot be worked out is simply not there; the board must never fail for the sake of an extra.
|
||||
console.error("[wall] couldn't work out the extra tiles:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
return { tiles, groups: buildGroups(connected, failing, alerts) };
|
||||
}
|
||||
|
||||
/** The extras for the signed-in wall, which has none of it read yet: the connected integrations, which kinds are failing, and the alerts there are. */
|
||||
export async function wallExtrasNow(nowMs: number = Date.now()): Promise<WallExtras> {
|
||||
const [rows, report] = await Promise.all([db.select({ id: integrations.id, type: integrations.type }).from(integrations).where(and(eq(integrations.enabled, true))), getAlerts(false)]);
|
||||
const failing = new Set(rows.filter((r) => getSourceStatus(r.type).status === "failing").map((r) => r.type));
|
||||
return buildWallExtras(rows, failing, report.alerts, nowMs);
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
/**
|
||||
* Runbooks for a KIND of alert (written once, shown beside every alert of that kind) and the wall display's extras: a tile for each connected thing worth a glance (Hetzner servers running, game
|
||||
* servers running, critical vulnerabilities, stuck downloads) and a summary for each group of integrations the menu has.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import type { Server } from "node:http";
|
||||
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-kinds-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, integrations, auditLog, dockhandVulnerabilities } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { runbooksRouter } = await import("../src/routes/runbooks.js");
|
||||
const { wallRouter } = await import("../src/routes/wall.js");
|
||||
const { kindOfKey } = await import("../src/services/alertTypes.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const hz = await import("../src/services/hetznerSync.js");
|
||||
const fn = await import("../src/services/fragnetSync.js");
|
||||
const arr = await import("../src/services/arrSync.js");
|
||||
const wall = await import("../src/services/wallExtras.js");
|
||||
const { INTEGRATION_GROUPS } = await import("../src/services/integrationGroups.js");
|
||||
const { buildPublicWall } = await import("../src/services/publicWall.js");
|
||||
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
const iso = (ms: number) => new Date(ms).toISOString();
|
||||
|
||||
describe("the kind of an alert", () => {
|
||||
test("is the first part of its key", () => {
|
||||
assert.equal(kindOfKey("disk:server:3:/"), "disk");
|
||||
assert.equal(kindOfKey("hetzner-cert:2:20"), "hetzner-cert");
|
||||
assert.equal(kindOfKey("sonarr-queue:7"), "sonarr-queue");
|
||||
assert.equal(kindOfKey("plain"), "plain");
|
||||
});
|
||||
});
|
||||
|
||||
describe("runbooks for a kind of alert", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const call = (sub: string, method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json", "x-sub": sub }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }, { oidcSub: "operator", name: "Olle", role: "operator" }]);
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: String(req.headers["x-sub"] ?? "viewer") } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/runbooks", runbooksRouter);
|
||||
a.use("/api/wall", wallRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
test("an operator writes one, everyone signed in reads the list; a viewer cannot write", async () => {
|
||||
assert.equal((await call("viewer", "PUT", "/runbooks/kinds/hetzner-cert", { title: "x", body: "y" })).status, 403);
|
||||
const put = await call("operator", "PUT", "/runbooks/kinds/hetzner-cert", { title: "If a certificate is about to end", body: "Renew it in the Hetzner console.\n\n- then check the load balancer", url: "https://wiki.example.lan/certs" });
|
||||
assert.equal(put.status, 200);
|
||||
const list = (await (await call("viewer", "GET", "/runbooks/kinds")).json()) as { kinds: { kind: string; title: string; body: string; url: string | null; updatedBy: string }[] };
|
||||
assert.deepEqual(list.kinds.map((k) => [k.kind, k.title, k.url, k.updatedBy]), [["hetzner-cert", "If a certificate is about to end", "https://wiki.example.lan/certs", "Olle"]]);
|
||||
assert.match(list.kinds[0].body, /Renew it in the Hetzner console/);
|
||||
});
|
||||
|
||||
test("writing it again changes it (one for each kind), with a link alone or a text alone", async () => {
|
||||
assert.equal((await call("operator", "PUT", "/runbooks/kinds/hetzner-cert", { title: "Certificates", body: "", url: "https://wiki.example.lan/certs" })).status, 200);
|
||||
assert.equal((await call("operator", "PUT", "/runbooks/kinds/disk", { title: "A disk is filling", body: "Check what grew: du -sh /*" })).status, 200);
|
||||
const list = (await (await call("viewer", "GET", "/runbooks/kinds")).json()) as { kinds: { kind: string; title: string; url: string | null }[] };
|
||||
assert.deepEqual(list.kinds.map((k) => [k.kind, k.title, k.url]), [["disk", "A disk is filling", null], ["hetzner-cert", "Certificates", "https://wiki.example.lan/certs"]]);
|
||||
});
|
||||
|
||||
test("it must say something, be titled, and a link must be a web address; a kind is letters, digits and dashes", async () => {
|
||||
assert.equal((await call("operator", "PUT", "/runbooks/kinds/disk", { title: "x", body: " " })).status, 400, "nothing to say");
|
||||
assert.equal((await call("operator", "PUT", "/runbooks/kinds/disk", { title: "", body: "y" })).status, 400, "no title");
|
||||
assert.equal((await call("operator", "PUT", "/runbooks/kinds/disk", { title: "x", body: "y", url: "javascript:alert(1)" })).status, 400, "not a web address");
|
||||
assert.equal((await call("operator", "PUT", "/runbooks/kinds/disk", { title: "x", body: "y", url: "ftp://example.lan/x" })).status, 400);
|
||||
assert.equal((await call("operator", "PUT", "/runbooks/kinds/a%20b", { title: "x", body: "y" })).status, 400, "a kind with a space");
|
||||
assert.equal((await call("operator", "PUT", `/runbooks/kinds/${"k".repeat(61)}`, { title: "x", body: "y" })).status, 400, "too long");
|
||||
const list = (await (await call("viewer", "GET", "/runbooks/kinds")).json()) as { kinds: { kind: string; title: string }[] };
|
||||
assert.equal(list.kinds.find((k) => k.kind === "disk")?.title, "A disk is filling", "what was refused changed nothing");
|
||||
});
|
||||
|
||||
test("it can be taken away by an operator, and says so when there is none; every change is in the audit log", async () => {
|
||||
assert.equal((await call("viewer", "DELETE", "/runbooks/kinds/disk")).status, 403);
|
||||
assert.equal((await call("operator", "DELETE", "/runbooks/kinds/disk")).status, 204);
|
||||
assert.equal((await call("operator", "DELETE", "/runbooks/kinds/disk")).status, 404);
|
||||
const entries = (await db.select().from(auditLog)).filter((e) => e.targetType === "alert_kind").map((e) => `${e.action}:${e.targetId}`);
|
||||
assert.deepEqual(entries, ["create:hetzner-cert", "update:hetzner-cert", "create:disk", "delete:disk"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("an alert says what kind and what source it is", () => {
|
||||
let id = 0;
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "sonarr", name: "TV", baseUrl: "http://sonarr", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
await arr.storeRead(id, { version: "4", health: [{ source: "DownloadClientCheck", type: "error", message: "No download client is available" }], queue: { total: 0, items: [] }, disks: [], missing: 0, indexers: null }, new Date());
|
||||
await setInternalFlag("arrSync", JSON.stringify({ [id]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
});
|
||||
after(async () => {
|
||||
await arr.forgetArr(id);
|
||||
await db.delete(integrations).where(eq(integrations.id, id));
|
||||
});
|
||||
|
||||
test("kind from the key, type from where it came from", async () => {
|
||||
invalidateAlertsCache();
|
||||
const a = (await getAlerts(true)).alerts.find((x) => x.id.includes(`sonarr-health:${id}:`))!;
|
||||
assert.deepEqual([a.kind, a.type], ["sonarr-health", "sonarr"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the wall's groups", () => {
|
||||
test("the groups are the ones the menu has: the server's list is the web's list", () => {
|
||||
const text = readFileSync(join(import.meta.dirname, "../../web/src/utils/integrationTypes.ts"), "utf8");
|
||||
const webGroups = [...text.matchAll(/\{ id: "([a-z]+)", label: "([A-Za-z]+)", types: \[([^\]]*)\] \}/g)].map((m) => ({ id: m[1], label: m[2], types: [...m[3].matchAll(/"([a-z0-9]+)"/g)].map((t) => t[1]) }));
|
||||
assert.ok(webGroups.length >= 6, "the web's groups were found");
|
||||
assert.equal(JSON.stringify(INTEGRATION_GROUPS), JSON.stringify(webGroups));
|
||||
});
|
||||
|
||||
const connected = [{ id: 1, type: "proxmox" }, { id: 2, type: "hetzner" }, { id: 3, type: "jellyfin" }, { id: 4, type: "sonarr" }, { id: 5, type: "legacy-kind" }];
|
||||
const alert = (type: string, severity: "critical" | "warning" | "info", silenced = false) => ({ type, severity, silenced });
|
||||
|
||||
test("a group with something connected is there, with how many are connected, failing and alerting; a group with nothing is not", () => {
|
||||
const g = wall.buildGroups(connected, new Set(["hetzner"]), [alert("proxmox", "critical"), alert("hetzner", "warning"), alert("hetzner", "warning"), alert("sonarr", "warning"), alert("jellyfin", "info")]);
|
||||
assert.deepEqual(g.map((x) => [x.id, x.integrations, x.failing, x.alerts.critical, x.alerts.warning]), [["infrastructure", 2, 1, 1, 2], ["apps", 2, 0, 0, 1]]);
|
||||
});
|
||||
|
||||
test("a silenced alert does not count, nor does one about something in no group", () => {
|
||||
const g = wall.buildGroups(connected, new Set(), [alert("proxmox", "critical", true), alert("server", "critical"), alert("legacy-kind", "warning")]);
|
||||
assert.deepEqual(g.map((x) => [x.id, x.alerts.critical, x.alerts.warning]), [["infrastructure", 0, 0], ["apps", 0, 0]]);
|
||||
});
|
||||
|
||||
test("running tiles: all running is ok; a stopped cloud server is a warning, a stopped game server only a note", () => {
|
||||
assert.deepEqual({ ...wall.runningTile("a", "x", 3, 3, "warn") }, { id: "a", label: "x", figure: "3/3", state: "ok" });
|
||||
assert.equal(wall.runningTile("a", "x", 2, 3, "warn").state, "warn");
|
||||
assert.equal(wall.runningTile("a", "x", 2, 3, "info").state, "info");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the wall's tiles, from what is stored", () => {
|
||||
const ids: Record<string, number> = {};
|
||||
const mk = async (type: "hetzner" | "fragnet" | "dockhand" | "sonarr", name: string) => (await db.insert(integrations).values({ type, name, baseUrl: "http://x", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
const fresh = async () => {
|
||||
const at = { at: new Date().toISOString(), ok: true, error: null };
|
||||
await setInternalFlag("hetznerSync", JSON.stringify({ [ids.hz]: at }));
|
||||
await setInternalFlag("fragnetSync", JSON.stringify({ [ids.fn]: at }));
|
||||
await setInternalFlag("dockerVulnSync", JSON.stringify({ [ids.dh]: at }));
|
||||
await setInternalFlag("arrSync", JSON.stringify({ [ids.sonarr]: at }));
|
||||
};
|
||||
const stale = async () => {
|
||||
const at = { at: new Date(Date.now() - 3 * HOUR).toISOString(), ok: true, error: null };
|
||||
await setInternalFlag("hetznerSync", JSON.stringify({ [ids.hz]: at }));
|
||||
await setInternalFlag("fragnetSync", JSON.stringify({ [ids.fn]: at }));
|
||||
await setInternalFlag("dockerVulnSync", JSON.stringify({ [ids.dh]: at }));
|
||||
await setInternalFlag("arrSync", JSON.stringify({ [ids.sonarr]: at }));
|
||||
};
|
||||
const connected = () => Object.values(ids).map((id, i) => ({ id, type: ["hetzner", "fragnet", "dockhand", "sonarr"][i] }));
|
||||
const tileOf = (t: Awaited<ReturnType<typeof wall.buildWallExtras>>, id: string) => t.tiles.find((x) => x.id === id);
|
||||
|
||||
before(async () => {
|
||||
ids.hz = await mk("hetzner", "Prod");
|
||||
ids.fn = await mk("fragnet", "Games");
|
||||
ids.dh = await mk("dockhand", "Docker");
|
||||
ids.sonarr = await mk("sonarr", "TV");
|
||||
const now = Date.now();
|
||||
await hz.storeRead(ids.hz, { servers: [{ id: 1, name: "web", status: "running", outgoingTraffic: null }, { id: 2, name: "db", status: "off", outgoingTraffic: null }], volumes: null, images: null, primaryIps: null, prices: null, loadBalancers: null, floatingIps: null, certificates: null } as never, new Date());
|
||||
await fn.storeRead(ids.fn, { servers: [{ id: "aaaa1111", name: "survival", state: "running", suspended: false }, { id: "bbbb2222", name: "rust", state: "offline", suspended: false }, { id: "cccc3333", name: "old", state: "offline", suspended: true }] } as never, new Date());
|
||||
await db.insert(dockhandVulnerabilities).values([
|
||||
{ integrationId: ids.dh, environmentId: 1, environmentName: "Jackie", critical: 3, high: 7, medium: 1, low: 0, imagesScanned: 5, totalImages: 5, findings: "[]", updatedAt: new Date().toISOString() },
|
||||
{ integrationId: ids.dh, environmentId: 2, environmentName: "Anna", critical: 2, high: 0, medium: 0, low: 0, imagesScanned: 2, totalImages: 2, findings: "[]", updatedAt: new Date().toISOString() },
|
||||
{ integrationId: ids.dh, environmentId: 3, environmentName: "Unknown", critical: null, high: null, medium: null, low: null, imagesScanned: null, totalImages: null, findings: "[]", updatedAt: new Date().toISOString() },
|
||||
]);
|
||||
const item = (id: number, extra: Record<string, unknown> = {}) => ({ id, title: `t${id}`, status: "downloading", tracked: "ok", state: "downloading", messages: [], size: 1, sizeLeft: 1, added: iso(now - 30 * MIN), etaAt: null, protocol: null, client: null, indexer: null, ...extra });
|
||||
await arr.storeRead(ids.sonarr, { version: "4", health: [], queue: { total: 3, items: [item(1), item(2, { tracked: "error", added: iso(now - 5 * HOUR) }), item(3, { state: "importBlocked", added: iso(now - 4 * HOUR) })] }, disks: [], missing: 0, indexers: null }, new Date());
|
||||
await fresh();
|
||||
});
|
||||
after(async () => {
|
||||
for (const id of Object.values(ids)) await db.delete(integrations).where(eq(integrations.id, id));
|
||||
await hz.forgetHetzner(ids.hz);
|
||||
await fn.forgetFragnet(ids.fn);
|
||||
await arr.forgetArr(ids.sonarr);
|
||||
});
|
||||
|
||||
test("Hetzner servers running (a stop is a warning), game servers running (a stop is a note, a suspended one is not counted), critical vulnerabilities (an unknown host adds nothing), stuck downloads", async () => {
|
||||
const x = await wall.buildWallExtras(connected(), new Set(), []);
|
||||
assert.deepEqual({ ...tileOf(x, "hetzner") }, { id: "hetzner", label: "Hetzner servers running", figure: "1/2", state: "warn" });
|
||||
assert.deepEqual({ ...tileOf(x, "game-servers") }, { id: "game-servers", label: "game servers running", figure: "1/2", state: "info" });
|
||||
assert.deepEqual({ ...tileOf(x, "vulnerabilities") }, { id: "vulnerabilities", label: "critical vulnerabilities", figure: "5", state: "bad" });
|
||||
assert.deepEqual({ ...tileOf(x, "stuck-downloads") }, { id: "stuck-downloads", label: "stuck downloads", figure: "2", state: "warn" });
|
||||
assert.deepEqual(x.groups.map((g) => g.id), ["infrastructure", "apps"]);
|
||||
});
|
||||
|
||||
test("a reading that is old is not shown as how things are; nothing connected, no tile", async () => {
|
||||
await stale();
|
||||
const x = await wall.buildWallExtras(connected(), new Set(), []);
|
||||
assert.deepEqual(x.tiles, []);
|
||||
await fresh();
|
||||
assert.deepEqual((await wall.buildWallExtras([], new Set(), [])).tiles, []);
|
||||
assert.deepEqual((await wall.buildWallExtras([{ id: ids.hz, type: "hetzner" }], new Set(), [])).tiles.map((t) => t.id), ["hetzner"], "only what is connected");
|
||||
});
|
||||
|
||||
test("the share link carries them at every level (numbers and group names only), and the signed-in wall gets them from its own route", async () => {
|
||||
for (const detail of ["summary", "names"] as const) {
|
||||
const w = await buildPublicWall(detail);
|
||||
assert.deepEqual(w.tiles.map((t) => t.id).sort(), ["game-servers", "hetzner", "stuck-downloads", "vulnerabilities"], detail);
|
||||
assert.deepEqual(w.groups.map((g) => g.label), ["Infrastructure", "Apps"], detail);
|
||||
assert.doesNotMatch(JSON.stringify([w.tiles, w.groups]), /web|db|survival|Jackie/, "no names of anything");
|
||||
}
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/wall", wallRouter);
|
||||
const server = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
try {
|
||||
const body = (await (await fetch(`http://127.0.0.1:${(server.address() as AddressInfo).port}/api/wall/extras`)).json()) as { tiles: { id: string }[]; groups: { id: string }[] };
|
||||
assert.deepEqual(body.tiles.map((t) => t.id).sort(), ["game-servers", "hetzner", "stuck-downloads", "vulnerabilities"]);
|
||||
assert.deepEqual(body.groups.map((g) => g.id), ["infrastructure", "apps"]);
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
@@ -152,6 +152,22 @@ describe("the fake Dockhand", () => {
|
||||
assert.deepEqual(dv.dockerVulnProblems(id, [{ ...jackie, critical: 0, high: 50 }]), []);
|
||||
});
|
||||
|
||||
test("high findings are raised only when a number is set, from that many, and then beside the critical ones", async () => {
|
||||
const jackie = (await rows()).find((r) => r.environmentName === "Jackie")!;
|
||||
const many = { ...jackie, critical: 0, high: 12 };
|
||||
assert.deepEqual(dv.dockerVulnProblems(id, [many], 0), [], "off by default");
|
||||
assert.deepEqual(dv.dockerVulnProblems(id, [many]), [], "and with no number given");
|
||||
assert.deepEqual(dv.dockerVulnProblems(id, [many], 13), [], "below the number");
|
||||
const p = dv.dockerVulnProblems(id, [many], 10);
|
||||
assert.deepEqual(p.map((x) => [x.key, x.severity]), [[`docker-vuln-high:${id}:1`, "warning"]]);
|
||||
assert.equal(p[0].message, "Jackie: 12 high vulnerabilities in the scanned images (you asked to hear of 10 or more)");
|
||||
assert.equal(dv.dockerVulnProblems(id, [{ ...many, high: 12 }], 12).length, 1, "at the number counts");
|
||||
assert.deepEqual(dv.dockerVulnProblems(id, [{ ...many, high: 1 }], 1).map((x) => x.message), ["Jackie: 1 high vulnerability in the scanned images (you asked to hear of 1 or more)"]);
|
||||
assert.deepEqual(dv.dockerVulnProblems(id, [{ ...many, high: null }], 1), [], "an unknown count says nothing");
|
||||
const both = dv.dockerVulnProblems(id, [{ ...jackie, critical: 2, high: 40 }], 10);
|
||||
assert.deepEqual(both.map((x) => x.key), [`docker-vuln-high:${id}:1`, `docker-vuln:${id}:1`]);
|
||||
});
|
||||
|
||||
describe("the health pass, the Alerts page, the Docker page and /metrics", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
@@ -224,6 +240,30 @@ describe("the fake Dockhand", () => {
|
||||
assert.equal((await getSettings()).healthChecks.vulnerabilityAlerts, true);
|
||||
});
|
||||
|
||||
test("high findings: off by default (an old setting without the number is read as 0); with a number the host's high count is an alert of its own, under maintenance like the other", async () => {
|
||||
const checks = (await getSettings()).healthChecks;
|
||||
assert.equal(checks.vulnerabilityHighThreshold, 0);
|
||||
const { vulnerabilityHighThreshold: _dropped, ...older } = checks;
|
||||
await updateSettings({ healthChecks: older as never });
|
||||
assert.equal((await getSettings()).healthChecks.vulnerabilityHighThreshold, 0);
|
||||
const highIds = async () => {
|
||||
invalidateAlertsCache();
|
||||
return (await getAlerts(true)).alerts.map((x) => x.id).filter((i) => i.includes("docker-vuln-high:"));
|
||||
};
|
||||
assert.deepEqual(await highIds(), []);
|
||||
await updateSettings({ healthChecks: { ...checks, vulnerabilityHighThreshold: 5 } });
|
||||
assert.ok((await highIds()).includes(`services:docker-vuln-high:${id}:1`), "Jackie has 7 high findings");
|
||||
const a = (await getAlerts(true)).alerts.find((x) => x.id === `services:docker-vuln-high:${id}:1`)!;
|
||||
assert.deepEqual([a.severity, a.source, a.link, a.target?.type], ["warning", "Docker", "/docker", "integration"]);
|
||||
assert.match(a.message, /Jackie: 7 high vulnerabilities/);
|
||||
assert.equal(subjectOfConditionKey(`docker-vuln-high:${id}:1`), `integration:${id}`);
|
||||
await updateSettings({ healthChecks: { ...checks, vulnerabilityHighThreshold: 8 } });
|
||||
assert.ok(!(await highIds()).includes(`services:docker-vuln-high:${id}:1`), "7 is under 8");
|
||||
await updateSettings({ healthChecks: { ...checks, vulnerabilityAlerts: false, vulnerabilityHighThreshold: 5 } });
|
||||
assert.deepEqual(await highIds(), [], "the switch for the vulnerability alerts turns this off as well");
|
||||
await updateSettings({ healthChecks: checks });
|
||||
});
|
||||
|
||||
test("anyone signed in sees each host's counts, the critical findings and how many images are scanned", async () => {
|
||||
const body = (await (await fetch(`${base}/vulnerabilities`, { headers: { connection: "close" } })).json()) as { alerts: boolean; fresh: boolean; environments: { environmentName: string; critical: number; high: number; imagesScanned: number; totalImages: number; findings: { cve: string }[] }[] };
|
||||
assert.deepEqual([body.alerts, body.fresh], [true, true]);
|
||||
|
||||
Reference in new issue
Block a user