Files
Homelab-manager/server/test/alertKindsAndWall.test.ts
T
bobbanandClaude Sonnet 5.5 3d9a2310bb Runbooks per alert kind, high-vulnerability threshold, wall display extras
A "What to do" note (text and/or link) can be written once for a whole kind of alert and
shows beside every alert of that kind; alerts now carry their kind and source type.
New optional setting: warn when a Docker host has N or more high vulnerabilities (off by
default). The wall display gains tiles (Hetzner and game servers running, critical
vulnerabilities, stuck downloads) and a summary line for each group of integrations,
also in share links. Migration 0074; docs and tests included.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-27 21:15:45 +01:00

253 lines
17 KiB
TypeScript

/**
* Runbooks for a KIND of alert (written once, shown beside every alert of that kind) and the wall display's extras: a tile for each connected thing worth a glance (Hetzner servers running, game
* servers running, critical vulnerabilities, stuck downloads) and a summary for each group of integrations the menu has.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import type { Server } from "node:http";
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-kinds-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, auditLog, dockhandVulnerabilities } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { runbooksRouter } = await import("../src/routes/runbooks.js");
const { wallRouter } = await import("../src/routes/wall.js");
const { kindOfKey } = await import("../src/services/alertTypes.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const hz = await import("../src/services/hetznerSync.js");
const fn = await import("../src/services/fragnetSync.js");
const arr = await import("../src/services/arrSync.js");
const wall = await import("../src/services/wallExtras.js");
const { INTEGRATION_GROUPS } = await import("../src/services/integrationGroups.js");
const { buildPublicWall } = await import("../src/services/publicWall.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const HOUR = 60 * MIN;
const iso = (ms: number) => new Date(ms).toISOString();
describe("the kind of an alert", () => {
test("is the first part of its key", () => {
assert.equal(kindOfKey("disk:server:3:/"), "disk");
assert.equal(kindOfKey("hetzner-cert:2:20"), "hetzner-cert");
assert.equal(kindOfKey("sonarr-queue:7"), "sonarr-queue");
assert.equal(kindOfKey("plain"), "plain");
});
});
describe("runbooks for a kind of alert", () => {
let app: Server;
let base = "";
const call = (sub: string, method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json", "x-sub": sub }, body: body === undefined ? undefined : JSON.stringify(body) });
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }, { oidcSub: "operator", name: "Olle", role: "operator" }]);
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: String(req.headers["x-sub"] ?? "viewer") } };
next();
});
a.use("/api/runbooks", runbooksRouter);
a.use("/api/wall", wallRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => app.close());
test("an operator writes one, everyone signed in reads the list; a viewer cannot write", async () => {
assert.equal((await call("viewer", "PUT", "/runbooks/kinds/hetzner-cert", { title: "x", body: "y" })).status, 403);
const put = await call("operator", "PUT", "/runbooks/kinds/hetzner-cert", { title: "If a certificate is about to end", body: "Renew it in the Hetzner console.\n\n- then check the load balancer", url: "https://wiki.example.lan/certs" });
assert.equal(put.status, 200);
const list = (await (await call("viewer", "GET", "/runbooks/kinds")).json()) as { kinds: { kind: string; title: string; body: string; url: string | null; updatedBy: string }[] };
assert.deepEqual(list.kinds.map((k) => [k.kind, k.title, k.url, k.updatedBy]), [["hetzner-cert", "If a certificate is about to end", "https://wiki.example.lan/certs", "Olle"]]);
assert.match(list.kinds[0].body, /Renew it in the Hetzner console/);
});
test("writing it again changes it (one for each kind), with a link alone or a text alone", async () => {
assert.equal((await call("operator", "PUT", "/runbooks/kinds/hetzner-cert", { title: "Certificates", body: "", url: "https://wiki.example.lan/certs" })).status, 200);
assert.equal((await call("operator", "PUT", "/runbooks/kinds/disk", { title: "A disk is filling", body: "Check what grew: du -sh /*" })).status, 200);
const list = (await (await call("viewer", "GET", "/runbooks/kinds")).json()) as { kinds: { kind: string; title: string; url: string | null }[] };
assert.deepEqual(list.kinds.map((k) => [k.kind, k.title, k.url]), [["disk", "A disk is filling", null], ["hetzner-cert", "Certificates", "https://wiki.example.lan/certs"]]);
});
test("it must say something, be titled, and a link must be a web address; a kind is letters, digits and dashes", async () => {
assert.equal((await call("operator", "PUT", "/runbooks/kinds/disk", { title: "x", body: " " })).status, 400, "nothing to say");
assert.equal((await call("operator", "PUT", "/runbooks/kinds/disk", { title: "", body: "y" })).status, 400, "no title");
assert.equal((await call("operator", "PUT", "/runbooks/kinds/disk", { title: "x", body: "y", url: "javascript:alert(1)" })).status, 400, "not a web address");
assert.equal((await call("operator", "PUT", "/runbooks/kinds/disk", { title: "x", body: "y", url: "ftp://example.lan/x" })).status, 400);
assert.equal((await call("operator", "PUT", "/runbooks/kinds/a%20b", { title: "x", body: "y" })).status, 400, "a kind with a space");
assert.equal((await call("operator", "PUT", `/runbooks/kinds/${"k".repeat(61)}`, { title: "x", body: "y" })).status, 400, "too long");
const list = (await (await call("viewer", "GET", "/runbooks/kinds")).json()) as { kinds: { kind: string; title: string }[] };
assert.equal(list.kinds.find((k) => k.kind === "disk")?.title, "A disk is filling", "what was refused changed nothing");
});
test("it can be taken away by an operator, and says so when there is none; every change is in the audit log", async () => {
assert.equal((await call("viewer", "DELETE", "/runbooks/kinds/disk")).status, 403);
assert.equal((await call("operator", "DELETE", "/runbooks/kinds/disk")).status, 204);
assert.equal((await call("operator", "DELETE", "/runbooks/kinds/disk")).status, 404);
const entries = (await db.select().from(auditLog)).filter((e) => e.targetType === "alert_kind").map((e) => `${e.action}:${e.targetId}`);
assert.deepEqual(entries, ["create:hetzner-cert", "update:hetzner-cert", "create:disk", "delete:disk"]);
});
});
describe("an alert says what kind and what source it is", () => {
let id = 0;
before(async () => {
id = (await db.insert(integrations).values({ type: "sonarr", name: "TV", baseUrl: "http://sonarr", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
await arr.storeRead(id, { version: "4", health: [{ source: "DownloadClientCheck", type: "error", message: "No download client is available" }], queue: { total: 0, items: [] }, disks: [], missing: 0, indexers: null }, new Date());
await setInternalFlag("arrSync", JSON.stringify({ [id]: { at: new Date().toISOString(), ok: true, error: null } }));
});
after(async () => {
await arr.forgetArr(id);
await db.delete(integrations).where(eq(integrations.id, id));
});
test("kind from the key, type from where it came from", async () => {
invalidateAlertsCache();
const a = (await getAlerts(true)).alerts.find((x) => x.id.includes(`sonarr-health:${id}:`))!;
assert.deepEqual([a.kind, a.type], ["sonarr-health", "sonarr"]);
});
});
describe("the wall's groups", () => {
test("the groups are the ones the menu has: the server's list is the web's list", () => {
const text = readFileSync(join(import.meta.dirname, "../../web/src/utils/integrationTypes.ts"), "utf8");
const webGroups = [...text.matchAll(/\{ id: "([a-z]+)", label: "([A-Za-z]+)", types: \[([^\]]*)\] \}/g)].map((m) => ({ id: m[1], label: m[2], types: [...m[3].matchAll(/"([a-z0-9]+)"/g)].map((t) => t[1]) }));
assert.ok(webGroups.length >= 6, "the web's groups were found");
assert.equal(JSON.stringify(INTEGRATION_GROUPS), JSON.stringify(webGroups));
});
const connected = [{ id: 1, type: "proxmox" }, { id: 2, type: "hetzner" }, { id: 3, type: "jellyfin" }, { id: 4, type: "sonarr" }, { id: 5, type: "legacy-kind" }];
const alert = (type: string, severity: "critical" | "warning" | "info", silenced = false) => ({ type, severity, silenced });
test("a group with something connected is there, with how many are connected, failing and alerting; a group with nothing is not", () => {
const g = wall.buildGroups(connected, new Set(["hetzner"]), [alert("proxmox", "critical"), alert("hetzner", "warning"), alert("hetzner", "warning"), alert("sonarr", "warning"), alert("jellyfin", "info")]);
assert.deepEqual(g.map((x) => [x.id, x.integrations, x.failing, x.alerts.critical, x.alerts.warning]), [["infrastructure", 2, 1, 1, 2], ["apps", 2, 0, 0, 1]]);
});
test("a silenced alert does not count, nor does one about something in no group", () => {
const g = wall.buildGroups(connected, new Set(), [alert("proxmox", "critical", true), alert("server", "critical"), alert("legacy-kind", "warning")]);
assert.deepEqual(g.map((x) => [x.id, x.alerts.critical, x.alerts.warning]), [["infrastructure", 0, 0], ["apps", 0, 0]]);
});
test("running tiles: all running is ok; a stopped cloud server is a warning, a stopped game server only a note", () => {
assert.deepEqual({ ...wall.runningTile("a", "x", 3, 3, "warn") }, { id: "a", label: "x", figure: "3/3", state: "ok" });
assert.equal(wall.runningTile("a", "x", 2, 3, "warn").state, "warn");
assert.equal(wall.runningTile("a", "x", 2, 3, "info").state, "info");
});
});
describe("the wall's tiles, from what is stored", () => {
const ids: Record<string, number> = {};
const mk = async (type: "hetzner" | "fragnet" | "dockhand" | "sonarr", name: string) => (await db.insert(integrations).values({ type, name, baseUrl: "http://x", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
const fresh = async () => {
const at = { at: new Date().toISOString(), ok: true, error: null };
await setInternalFlag("hetznerSync", JSON.stringify({ [ids.hz]: at }));
await setInternalFlag("fragnetSync", JSON.stringify({ [ids.fn]: at }));
await setInternalFlag("dockerVulnSync", JSON.stringify({ [ids.dh]: at }));
await setInternalFlag("arrSync", JSON.stringify({ [ids.sonarr]: at }));
};
const stale = async () => {
const at = { at: new Date(Date.now() - 3 * HOUR).toISOString(), ok: true, error: null };
await setInternalFlag("hetznerSync", JSON.stringify({ [ids.hz]: at }));
await setInternalFlag("fragnetSync", JSON.stringify({ [ids.fn]: at }));
await setInternalFlag("dockerVulnSync", JSON.stringify({ [ids.dh]: at }));
await setInternalFlag("arrSync", JSON.stringify({ [ids.sonarr]: at }));
};
const connected = () => Object.values(ids).map((id, i) => ({ id, type: ["hetzner", "fragnet", "dockhand", "sonarr"][i] }));
const tileOf = (t: Awaited<ReturnType<typeof wall.buildWallExtras>>, id: string) => t.tiles.find((x) => x.id === id);
before(async () => {
ids.hz = await mk("hetzner", "Prod");
ids.fn = await mk("fragnet", "Games");
ids.dh = await mk("dockhand", "Docker");
ids.sonarr = await mk("sonarr", "TV");
const now = Date.now();
await hz.storeRead(ids.hz, { servers: [{ id: 1, name: "web", status: "running", outgoingTraffic: null }, { id: 2, name: "db", status: "off", outgoingTraffic: null }], volumes: null, images: null, primaryIps: null, prices: null, loadBalancers: null, floatingIps: null, certificates: null } as never, new Date());
await fn.storeRead(ids.fn, { servers: [{ id: "aaaa1111", name: "survival", state: "running", suspended: false }, { id: "bbbb2222", name: "rust", state: "offline", suspended: false }, { id: "cccc3333", name: "old", state: "offline", suspended: true }] } as never, new Date());
await db.insert(dockhandVulnerabilities).values([
{ integrationId: ids.dh, environmentId: 1, environmentName: "Jackie", critical: 3, high: 7, medium: 1, low: 0, imagesScanned: 5, totalImages: 5, findings: "[]", updatedAt: new Date().toISOString() },
{ integrationId: ids.dh, environmentId: 2, environmentName: "Anna", critical: 2, high: 0, medium: 0, low: 0, imagesScanned: 2, totalImages: 2, findings: "[]", updatedAt: new Date().toISOString() },
{ integrationId: ids.dh, environmentId: 3, environmentName: "Unknown", critical: null, high: null, medium: null, low: null, imagesScanned: null, totalImages: null, findings: "[]", updatedAt: new Date().toISOString() },
]);
const item = (id: number, extra: Record<string, unknown> = {}) => ({ id, title: `t${id}`, status: "downloading", tracked: "ok", state: "downloading", messages: [], size: 1, sizeLeft: 1, added: iso(now - 30 * MIN), etaAt: null, protocol: null, client: null, indexer: null, ...extra });
await arr.storeRead(ids.sonarr, { version: "4", health: [], queue: { total: 3, items: [item(1), item(2, { tracked: "error", added: iso(now - 5 * HOUR) }), item(3, { state: "importBlocked", added: iso(now - 4 * HOUR) })] }, disks: [], missing: 0, indexers: null }, new Date());
await fresh();
});
after(async () => {
for (const id of Object.values(ids)) await db.delete(integrations).where(eq(integrations.id, id));
await hz.forgetHetzner(ids.hz);
await fn.forgetFragnet(ids.fn);
await arr.forgetArr(ids.sonarr);
});
test("Hetzner servers running (a stop is a warning), game servers running (a stop is a note, a suspended one is not counted), critical vulnerabilities (an unknown host adds nothing), stuck downloads", async () => {
const x = await wall.buildWallExtras(connected(), new Set(), []);
assert.deepEqual({ ...tileOf(x, "hetzner") }, { id: "hetzner", label: "Hetzner servers running", figure: "1/2", state: "warn" });
assert.deepEqual({ ...tileOf(x, "game-servers") }, { id: "game-servers", label: "game servers running", figure: "1/2", state: "info" });
assert.deepEqual({ ...tileOf(x, "vulnerabilities") }, { id: "vulnerabilities", label: "critical vulnerabilities", figure: "5", state: "bad" });
assert.deepEqual({ ...tileOf(x, "stuck-downloads") }, { id: "stuck-downloads", label: "stuck downloads", figure: "2", state: "warn" });
assert.deepEqual(x.groups.map((g) => g.id), ["infrastructure", "apps"]);
});
test("a reading that is old is not shown as how things are; nothing connected, no tile", async () => {
await stale();
const x = await wall.buildWallExtras(connected(), new Set(), []);
assert.deepEqual(x.tiles, []);
await fresh();
assert.deepEqual((await wall.buildWallExtras([], new Set(), [])).tiles, []);
assert.deepEqual((await wall.buildWallExtras([{ id: ids.hz, type: "hetzner" }], new Set(), [])).tiles.map((t) => t.id), ["hetzner"], "only what is connected");
});
test("the share link carries them at every level (numbers and group names only), and the signed-in wall gets them from its own route", async () => {
for (const detail of ["summary", "names"] as const) {
const w = await buildPublicWall(detail);
assert.deepEqual(w.tiles.map((t) => t.id).sort(), ["game-servers", "hetzner", "stuck-downloads", "vulnerabilities"], detail);
assert.deepEqual(w.groups.map((g) => g.label), ["Infrastructure", "Apps"], detail);
assert.doesNotMatch(JSON.stringify([w.tiles, w.groups]), /web|db|survival|Jackie/, "no names of anything");
}
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/wall", wallRouter);
const server = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
try {
const body = (await (await fetch(`http://127.0.0.1:${(server.address() as AddressInfo).port}/api/wall/extras`)).json()) as { tiles: { id: string }[]; groups: { id: string }[] };
assert.deepEqual(body.tiles.map((t) => t.id).sort(), ["game-servers", "hetzner", "stuck-downloads", "vulnerabilities"]);
assert.deepEqual(body.groups.map((g) => g.id), ["infrastructure", "apps"]);
} finally {
server.close();
}
});
});
after(() => {
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});