Files
Homelab-manager/server/test/alertTimeline.test.ts
T
bobbanandClaude Sonnet 5.5 461219844b Say who was told in an alert's timeline
Notifications are written down with the alerts they were about, and the
incident timeline lists each one: which channels took it, which refused and
what they said, or why it was held or not sent. New alerts, recovery notices,
reminders and escalations are all covered.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-10 21:14:58 +01:00

171 lines
12 KiB
TypeScript

/**
* Who was told, in an alert's timeline: a notification is written down with the alerts it was about, and the incident's timeline lists each one that was about its alert — how each channel took it, or why it
* was held or not sent — alongside the acknowledgements, maintenance and the rest.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-alerttimeline-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const { db } = await import("../src/db/client.js");
const { alertIncidents } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const log = await import("../src/services/notificationLog.js");
const inc = await import("../src/services/incidents.js");
const { notifyHealthIssues, notifyHealthRecovered, notifyAudience } = await import("../src/services/notify.js");
const { updateSettings } = await import("../src/services/settingsStore.js");
await runMigrations();
const MIN = 60_000;
const NOW = Date.parse("2026-11-10T12:00:00Z");
const iso = (ms: number) => new Date(ms).toISOString();
const ok = (channel: string) => ({ channel, ok: true, error: null });
const bad = (channel: string, error: string) => ({ channel, ok: false, error });
describe("how a notification reads in the timeline", () => {
const d = inc.describeNotification;
test("sent: who it reached, in words, with what kind of message it was", () => {
const e = d({ title: "Homelab Manager — Health Alert", status: "sent", note: null, results: [ok("ntfy"), ok("telegram"), ok("smtp")] });
assert.deepEqual([e.kind, e.tone, e.title, e.detail], ["notify", "good", "Told through ntfy, Telegram and email", "Health Alert"]);
assert.equal(d({ title: "T", status: "sent", note: null, results: [ok("webPush")] }).title, "Told through push");
});
test("partly sent says who did not take it and what they said; failed says nobody could be told", () => {
const partly = d({ title: "Homelab Manager — Health Alert", status: "partly", note: null, results: [ok("ntfy"), bad("gotify", "Gotify error 401: unauthorised")] });
assert.deepEqual([partly.tone, partly.title], ["bad", "Told through ntfy — Gotify did not take it"]);
assert.match(partly.detail ?? "", /Gotify: Gotify error 401: unauthorised/);
const failed = d({ title: "Homelab Manager — Health Recovered", status: "failed", note: null, results: [bad("ntfy", "ntfy failed: fetch failed")] });
assert.deepEqual([failed.tone, failed.title], ["bad", "Nobody could be told — every channel failed"]);
assert.match(failed.detail ?? "", /Health Recovered · ntfy: ntfy failed/);
});
test("held, muted and nowhere to send each say so, in a neutral tone", () => {
assert.deepEqual([d({ title: "T", status: "held", note: "Held for quiet hours.", results: [] }).tone, d({ title: "T", status: "held", note: "Held for quiet hours.", results: [] }).title], ["neutral", "The message was held back"]);
assert.match(d({ title: "T", status: "held", note: "Held for quiet hours.", results: [] }).detail ?? "", /Held for quiet hours/);
assert.match(d({ title: "T", status: "muted", note: null, results: [] }).title, /routing rule says nobody/);
assert.match(d({ title: "T", status: "none", note: null, results: [] }).title, /no channel is switched on/);
});
});
describe("what a notification says it was about", () => {
test("the keys are written with it, once each, up to fifty, and read back as a list", async () => {
const id = (await log.recordNotification({ title: "t", message: "m", results: [], keys: ["a:1", "a:1", "b:2", ""] }))!;
const row = await log.getLogged(id);
assert.deepEqual(log.keysOf(row!), ["a:1", "b:2"]);
const many = (await log.recordNotification({ title: "t", message: "m", results: [], keys: Array.from({ length: 80 }, (_, i) => `k:${i}`) }))!;
assert.equal(log.keysOf((await log.getLogged(many))!).length, 50);
assert.deepEqual(log.keysOf({ alertKeys: null }), []);
for (const junk of ["nope", "{}", "5"]) assert.deepEqual(log.keysOf({ alertKeys: junk }), []);
});
test("an alert's notifications are found by its key and its time, and only its own", async () => {
await log.recordNotification({ title: "one", message: "m", results: [ok("ntfy")], keys: ["disk:server:3:/", "offline:server:4"] }, new Date(NOW - 10 * MIN));
await log.recordNotification({ title: "two", message: "m", results: [ok("ntfy")], keys: ["disk:server:3:/"] }, new Date(NOW - 3 * 60 * MIN));
await log.recordNotification({ title: "three", message: "m", results: [ok("ntfy")], keys: ["disk:server:33:/"] }, new Date(NOW - 5 * MIN));
await log.recordNotification({ title: "four", message: "m", results: [ok("ntfy")] }, new Date(NOW - 5 * MIN));
const found = await log.loggedForAlert("disk:server:3:/", iso(NOW - 60 * MIN), iso(NOW));
assert.deepEqual(found.map((r) => r.title), ["one"], "not the one three hours ago, not another server's, not one about nothing");
assert.deepEqual((await log.loggedForAlert("disk:server:3:/", iso(NOW - 4 * 60 * MIN), iso(NOW))).map((r) => r.title), ["two", "one"]);
});
test("a key with a character LIKE treats as a wildcard does not find another alert's", async () => {
await log.recordNotification({ title: "underscore", message: "m", results: [], keys: ["a_b:1"] }, new Date(NOW - MIN));
await log.recordNotification({ title: "other", message: "m", results: [], keys: ["axb:1"] }, new Date(NOW - MIN));
assert.deepEqual((await log.loggedForAlert("a_b:1", iso(NOW - 10 * MIN), iso(NOW))).map((r) => r.title), ["underscore"]);
});
});
describe("through the notifications that are sent", () => {
let hook: Server;
let url = "";
const got: string[] = [];
before(async () => {
hook = createHttpServer((req, res) => {
req.resume();
got.push(req.url ?? "");
res.writeHead(200).end("ok");
});
await new Promise<void>((resolve) => hook.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(hook.address() as AddressInfo).port}/hook`;
await updateSettings({ webhook: { enabled: true, url }, ntfy: { enabled: false } });
});
after(async () => {
hook.close();
await updateSettings({ webhook: { enabled: false }, routing: { rules: [] }, quietHours: { enabled: false, start: "22:00", end: "07:00", criticalThrough: false } });
});
const about = async (key: string) => log.loggedForAlert(key, "2000-01-01T00:00:00.000Z", "2100-01-01T00:00:00.000Z");
test("a new alert and its recovery are each written down with the alert they were about", async () => {
await notifyHealthIssues([{ message: "web-1 is down", key: "offline:server:9", severity: "critical" }, { message: "disk on web-1", key: "disk:server:9:/", severity: "warning" }]);
await notifyHealthRecovered([{ message: "web-1 is down", key: "offline:server:9", severity: "critical" }]);
const mine = await about("offline:server:9");
assert.deepEqual(mine.map((r) => [r.title.replace("Homelab Manager — ", ""), r.status]), [["Health Alert", "sent"], ["Health Recovered", "sent"]]);
assert.deepEqual((await about("disk:server:9:/")).map((r) => r.title.replace("Homelab Manager — ", "")), ["Health Alert"], "the disk alert was in the first message only");
assert.ok(got.length >= 2, "and they were really sent");
});
test("one that a routing rule silenced, or quiet hours held, says so against the alert too", async () => {
await updateSettings({ routing: { rules: [{ id: "q", name: "no disks", enabled: true, categories: ["disk"], minSeverity: null, tags: [], channels: [] }] } });
await notifyHealthIssues([{ message: "disk filling", key: "disk:server:10:/", severity: "warning" }]);
assert.deepEqual((await about("disk:server:10:/")).map((r) => r.status), ["muted"]);
await updateSettings({ routing: { rules: [] } });
const h = new Date().getUTCHours();
const pad = (n: number) => String(((n % 24) + 24) % 24).padStart(2, "0") + ":00";
await updateSettings({ quietHours: { enabled: true, start: pad(h - 1), end: pad(h + 1), criticalThrough: false }, notifications: { timezone: "UTC" } as never });
await notifyAudience("Homelab Manager — Health Alert", "held", { channels: null, pushIds: null }, { keys: ["updates:server:9"] });
assert.deepEqual((await about("updates:server:9")).map((r) => r.status), ["held"]);
await updateSettings({ quietHours: { enabled: false, start: "22:00", end: "07:00", criticalThrough: false } });
});
});
describe("in the incident's timeline", () => {
let incidentId = 0;
before(async () => {
incidentId = (
await db
.insert(alertIncidents)
.values({ alertKey: "offline:server:77", severity: "critical", peakSeverity: "critical", category: "offline", source: "Servers", message: "pve9 has not reported", link: null, targetType: null, targetId: null, targetName: null, causedBy: null, openedAt: iso(NOW - 60 * MIN), lastSeenAt: iso(NOW - 5 * MIN), resolvedAt: iso(NOW - 20 * MIN) })
.returning({ id: alertIncidents.id })
)[0].id;
await log.recordNotification({ title: "Homelab Manager — Health Alert", message: "m", results: [ok("ntfy"), bad("gotify", "Gotify error 401: nope")], keys: ["offline:server:77"] }, new Date(NOW - 59 * MIN));
await log.recordNotification({ title: "Homelab Manager — Still not acknowledged", message: "m", results: [ok("telegram")], keys: ["offline:server:77"] }, new Date(NOW - 40 * MIN));
await log.recordNotification({ title: "Homelab Manager — Health Recovered", message: "m", results: [], status: "held", note: "Held for quiet hours.", keys: ["offline:server:77"] }, new Date(NOW - 18 * MIN));
await log.recordNotification({ title: "Homelab Manager — Health Alert", message: "m", results: [ok("ntfy")], keys: ["offline:server:78"] }, new Date(NOW - 59 * MIN));
await log.recordNotification({ title: "Homelab Manager — Health Alert", message: "m", results: [ok("ntfy")], keys: ["offline:server:77"] }, new Date(NOW - 3 * 60 * MIN));
});
after(() => {
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("each notification about the alert is an event, in order among the others, and nobody else's is", async () => {
const detail = (await inc.getIncident(incidentId, new Date(NOW)))!;
const told = detail.timeline.filter((e) => e.kind === "notify");
assert.deepEqual(told.map((e) => e.title), ["Told through ntfy — Gotify did not take it", "Told through Telegram", "The message was held back"]);
assert.deepEqual(told.map((e) => e.tone), ["bad", "good", "neutral"]);
assert.match(told[1].detail ?? "", /Still not acknowledged/);
assert.deepEqual(detail.timeline.map((e) => e.at), [...detail.timeline.map((e) => e.at)].sort(), "the whole timeline is in time order");
assert.deepEqual(detail.timeline.filter((e) => e.kind === "alert").map((e) => e.title), ["Alert raised (critical)", "Resolved"]);
});
test("a notification from long before the alert, or about another one, is left out", async () => {
const detail = (await inc.getIncident(incidentId, new Date(NOW)))!;
assert.equal(detail.timeline.filter((e) => e.kind === "notify").length, 3);
});
});