/** * Who was told, in an alert's timeline: a notification is written down with the alerts it was about, and the incident's timeline lists each one that was about its alert — how each channel took it, or why it * was held or not sent — alongside the acknowledgements, maintenance and the rest. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { createServer as createHttpServer, type Server } from "node:http"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-alerttimeline-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const { db } = await import("../src/db/client.js"); const { alertIncidents } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const log = await import("../src/services/notificationLog.js"); const inc = await import("../src/services/incidents.js"); const { notifyHealthIssues, notifyHealthRecovered, notifyAudience } = await import("../src/services/notify.js"); const { updateSettings } = await import("../src/services/settingsStore.js"); await runMigrations(); const MIN = 60_000; const NOW = Date.parse("2026-11-10T12:00:00Z"); const iso = (ms: number) => new Date(ms).toISOString(); const ok = (channel: string) => ({ channel, ok: true, error: null }); const bad = (channel: string, error: string) => ({ channel, ok: false, error }); describe("how a notification reads in the timeline", () => { const d = inc.describeNotification; test("sent: who it reached, in words, with what kind of message it was", () => { const e = d({ title: "Homelab Manager — Health Alert", status: "sent", note: null, results: [ok("ntfy"), ok("telegram"), ok("smtp")] }); assert.deepEqual([e.kind, e.tone, e.title, e.detail], ["notify", "good", "Told through ntfy, Telegram and email", "Health Alert"]); assert.equal(d({ title: "T", status: "sent", note: null, results: [ok("webPush")] }).title, "Told through push"); }); test("partly sent says who did not take it and what they said; failed says nobody could be told", () => { const partly = d({ title: "Homelab Manager — Health Alert", status: "partly", note: null, results: [ok("ntfy"), bad("gotify", "Gotify error 401: unauthorised")] }); assert.deepEqual([partly.tone, partly.title], ["bad", "Told through ntfy — Gotify did not take it"]); assert.match(partly.detail ?? "", /Gotify: Gotify error 401: unauthorised/); const failed = d({ title: "Homelab Manager — Health Recovered", status: "failed", note: null, results: [bad("ntfy", "ntfy failed: fetch failed")] }); assert.deepEqual([failed.tone, failed.title], ["bad", "Nobody could be told — every channel failed"]); assert.match(failed.detail ?? "", /Health Recovered · ntfy: ntfy failed/); }); test("held, muted and nowhere to send each say so, in a neutral tone", () => { assert.deepEqual([d({ title: "T", status: "held", note: "Held for quiet hours.", results: [] }).tone, d({ title: "T", status: "held", note: "Held for quiet hours.", results: [] }).title], ["neutral", "The message was held back"]); assert.match(d({ title: "T", status: "held", note: "Held for quiet hours.", results: [] }).detail ?? "", /Held for quiet hours/); assert.match(d({ title: "T", status: "muted", note: null, results: [] }).title, /routing rule says nobody/); assert.match(d({ title: "T", status: "none", note: null, results: [] }).title, /no channel is switched on/); }); }); describe("what a notification says it was about", () => { test("the keys are written with it, once each, up to fifty, and read back as a list", async () => { const id = (await log.recordNotification({ title: "t", message: "m", results: [], keys: ["a:1", "a:1", "b:2", ""] }))!; const row = await log.getLogged(id); assert.deepEqual(log.keysOf(row!), ["a:1", "b:2"]); const many = (await log.recordNotification({ title: "t", message: "m", results: [], keys: Array.from({ length: 80 }, (_, i) => `k:${i}`) }))!; assert.equal(log.keysOf((await log.getLogged(many))!).length, 50); assert.deepEqual(log.keysOf({ alertKeys: null }), []); for (const junk of ["nope", "{}", "5"]) assert.deepEqual(log.keysOf({ alertKeys: junk }), []); }); test("an alert's notifications are found by its key and its time, and only its own", async () => { await log.recordNotification({ title: "one", message: "m", results: [ok("ntfy")], keys: ["disk:server:3:/", "offline:server:4"] }, new Date(NOW - 10 * MIN)); await log.recordNotification({ title: "two", message: "m", results: [ok("ntfy")], keys: ["disk:server:3:/"] }, new Date(NOW - 3 * 60 * MIN)); await log.recordNotification({ title: "three", message: "m", results: [ok("ntfy")], keys: ["disk:server:33:/"] }, new Date(NOW - 5 * MIN)); await log.recordNotification({ title: "four", message: "m", results: [ok("ntfy")] }, new Date(NOW - 5 * MIN)); const found = await log.loggedForAlert("disk:server:3:/", iso(NOW - 60 * MIN), iso(NOW)); assert.deepEqual(found.map((r) => r.title), ["one"], "not the one three hours ago, not another server's, not one about nothing"); assert.deepEqual((await log.loggedForAlert("disk:server:3:/", iso(NOW - 4 * 60 * MIN), iso(NOW))).map((r) => r.title), ["two", "one"]); }); test("a key with a character LIKE treats as a wildcard does not find another alert's", async () => { await log.recordNotification({ title: "underscore", message: "m", results: [], keys: ["a_b:1"] }, new Date(NOW - MIN)); await log.recordNotification({ title: "other", message: "m", results: [], keys: ["axb:1"] }, new Date(NOW - MIN)); assert.deepEqual((await log.loggedForAlert("a_b:1", iso(NOW - 10 * MIN), iso(NOW))).map((r) => r.title), ["underscore"]); }); }); describe("through the notifications that are sent", () => { let hook: Server; let url = ""; const got: string[] = []; before(async () => { hook = createHttpServer((req, res) => { req.resume(); got.push(req.url ?? ""); res.writeHead(200).end("ok"); }); await new Promise((resolve) => hook.listen(0, "127.0.0.1", resolve)); url = `http://127.0.0.1:${(hook.address() as AddressInfo).port}/hook`; await updateSettings({ webhook: { enabled: true, url }, ntfy: { enabled: false } }); }); after(async () => { hook.close(); await updateSettings({ webhook: { enabled: false }, routing: { rules: [] }, quietHours: { enabled: false, start: "22:00", end: "07:00", criticalThrough: false } }); }); const about = async (key: string) => log.loggedForAlert(key, "2000-01-01T00:00:00.000Z", "2100-01-01T00:00:00.000Z"); test("a new alert and its recovery are each written down with the alert they were about", async () => { await notifyHealthIssues([{ message: "web-1 is down", key: "offline:server:9", severity: "critical" }, { message: "disk on web-1", key: "disk:server:9:/", severity: "warning" }]); await notifyHealthRecovered([{ message: "web-1 is down", key: "offline:server:9", severity: "critical" }]); const mine = await about("offline:server:9"); assert.deepEqual(mine.map((r) => [r.title.replace("Homelab Manager — ", ""), r.status]), [["Health Alert", "sent"], ["Health Recovered", "sent"]]); assert.deepEqual((await about("disk:server:9:/")).map((r) => r.title.replace("Homelab Manager — ", "")), ["Health Alert"], "the disk alert was in the first message only"); assert.ok(got.length >= 2, "and they were really sent"); }); test("one that a routing rule silenced, or quiet hours held, says so against the alert too", async () => { await updateSettings({ routing: { rules: [{ id: "q", name: "no disks", enabled: true, categories: ["disk"], minSeverity: null, tags: [], channels: [] }] } }); await notifyHealthIssues([{ message: "disk filling", key: "disk:server:10:/", severity: "warning" }]); assert.deepEqual((await about("disk:server:10:/")).map((r) => r.status), ["muted"]); await updateSettings({ routing: { rules: [] } }); const h = new Date().getUTCHours(); const pad = (n: number) => String(((n % 24) + 24) % 24).padStart(2, "0") + ":00"; await updateSettings({ quietHours: { enabled: true, start: pad(h - 1), end: pad(h + 1), criticalThrough: false }, notifications: { timezone: "UTC" } as never }); await notifyAudience("Homelab Manager — Health Alert", "held", { channels: null, pushIds: null }, { keys: ["updates:server:9"] }); assert.deepEqual((await about("updates:server:9")).map((r) => r.status), ["held"]); await updateSettings({ quietHours: { enabled: false, start: "22:00", end: "07:00", criticalThrough: false } }); }); }); describe("in the incident's timeline", () => { let incidentId = 0; before(async () => { incidentId = ( await db .insert(alertIncidents) .values({ alertKey: "offline:server:77", severity: "critical", peakSeverity: "critical", category: "offline", source: "Servers", message: "pve9 has not reported", link: null, targetType: null, targetId: null, targetName: null, causedBy: null, openedAt: iso(NOW - 60 * MIN), lastSeenAt: iso(NOW - 5 * MIN), resolvedAt: iso(NOW - 20 * MIN) }) .returning({ id: alertIncidents.id }) )[0].id; await log.recordNotification({ title: "Homelab Manager — Health Alert", message: "m", results: [ok("ntfy"), bad("gotify", "Gotify error 401: nope")], keys: ["offline:server:77"] }, new Date(NOW - 59 * MIN)); await log.recordNotification({ title: "Homelab Manager — Still not acknowledged", message: "m", results: [ok("telegram")], keys: ["offline:server:77"] }, new Date(NOW - 40 * MIN)); await log.recordNotification({ title: "Homelab Manager — Health Recovered", message: "m", results: [], status: "held", note: "Held for quiet hours.", keys: ["offline:server:77"] }, new Date(NOW - 18 * MIN)); await log.recordNotification({ title: "Homelab Manager — Health Alert", message: "m", results: [ok("ntfy")], keys: ["offline:server:78"] }, new Date(NOW - 59 * MIN)); await log.recordNotification({ title: "Homelab Manager — Health Alert", message: "m", results: [ok("ntfy")], keys: ["offline:server:77"] }, new Date(NOW - 3 * 60 * MIN)); }); after(() => { try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("each notification about the alert is an event, in order among the others, and nobody else's is", async () => { const detail = (await inc.getIncident(incidentId, new Date(NOW)))!; const told = detail.timeline.filter((e) => e.kind === "notify"); assert.deepEqual(told.map((e) => e.title), ["Told through ntfy — Gotify did not take it", "Told through Telegram", "The message was held back"]); assert.deepEqual(told.map((e) => e.tone), ["bad", "good", "neutral"]); assert.match(told[1].detail ?? "", /Still not acknowledged/); assert.deepEqual(detail.timeline.map((e) => e.at), [...detail.timeline.map((e) => e.at)].sort(), "the whole timeline is in time order"); assert.deepEqual(detail.timeline.filter((e) => e.kind === "alert").map((e) => e.title), ["Alert raised (critical)", "Resolved"]); }); test("a notification from long before the alert, or about another one, is left out", async () => { const detail = (await inc.getIncident(incidentId, new Date(NOW)))!; assert.equal(detail.timeline.filter((e) => e.kind === "notify").length, 3); }); });