/** * Closing an alert: hidden with no end, until it gets worse or has been gone for half an hour; why it was closed goes on its incident; it can be opened again; and how it sits * beside snoozing. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; import type { Server } from "node:http"; const dir = mkdtempSync(join(tmpdir(), "hm-close-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, heartbeats, alertSnoozes, auditLog } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { alertsRouter } = await import("../src/routes/alerts.js"); const { alertsBulkRouter } = await import("../src/routes/bulk.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { setClosed, setSnooze, settleSnoozes, withoutSnoozed, loadSnoozes, GONE_AFTER_MS, CLOSED_UNTIL } = await import("../src/services/alertSnoozes.js"); const { getIncident, openIncidentIds } = await import("../src/services/incidents.js"); const { getChangeLog } = await import("../src/services/changeLog.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); let app: Server; let base = ""; let who = "operator"; const call = async (method: string, path: string, body?: unknown) => { const res = await fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); return { status: res.status, body: (await res.json().catch(() => null)) as any }; }; const idOfHeartbeat = async (name: string) => { invalidateAlertsCache(); const report = await getAlerts(true); const a = report.alerts.find((x) => x.message.startsWith(name)); assert.ok(a, `an alert about ${name}`); return a!; }; before(async () => { await db.insert(users).values([ { oidcSub: "operator", name: "Olga", role: "operator" }, { oidcSub: "viewer", name: "Vic", role: "viewer" }, ]); const old = "2026-01-01 00:00:00"; for (const name of ["jobA", "jobB"]) await db.insert(heartbeats).values({ name, token: `hb_${name}`.padEnd(24, "x"), periodMinutes: 60, graceMinutes: 5, serverId: null, createdAt: old }); const a = express(); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/alerts/bulk", alertsBulkRouter); a.use("/api/alerts", alertsRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/alerts`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); describe("closing through the route", () => { test("a closed alert is held back like a snoozed one, with who closed it and why; it is not counted as needing attention", async () => { const a = await idOfHeartbeat("jobA"); assert.equal(a.silenced, false); const attention = (c: { critical: number; warning: number; info: number }) => c.critical + c.warning + c.info; const before = (await getAlerts(false)).counts; const r = await call("PUT", "/close", { alertId: a.id, note: " Replaced the job; the old one is retired. " }); assert.equal(r.status, 200); assert.deepEqual([r.body.closed.by, r.body.closed.note], ["Olga", "Replaced the job; the old one is retired."]); invalidateAlertsCache(); const report = await getAlerts(true); const after = report.alerts.find((x) => x.id === a.id)!; assert.deepEqual([after.silenced, after.closed?.by, after.closed?.note, after.snoozed], [true, "Olga", "Replaced the job; the old one is retired.", undefined]); assert.equal(attention(report.counts), attention(before) - 1, "no longer among those that need a look"); assert.equal(report.counts.silenced, before.silenced + 1, "counted as held back"); }); test("the reason is written on the incident, and opening it again says so there too", async () => { const a = await idOfHeartbeat("jobA"); const incidentId = (await openIncidentIds([a.id])).get(a.id)!; assert.ok(incidentId, "an incident is open for it"); let notes = (await getIncident(incidentId))!.notes.map((n) => [n.author, n.body]); assert.deepEqual(notes, [["Olga", "Closed: Replaced the job; the old one is retired."]]); assert.equal((await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`)).status, 204); notes = (await getIncident(incidentId))!.notes.map((n) => [n.author, n.body]); assert.deepEqual(notes.at(-1), ["Olga", "Opened again."]); invalidateAlertsCache(); const again = (await getAlerts(true)).alerts.find((x) => x.id === a.id)!; assert.deepEqual([again.silenced, again.closed], [false, undefined], "shown and counted again"); }); test("it is in the audit log and the change log as sentences; closing without a note says so on the incident", async () => { const a = await idOfHeartbeat("jobB"); assert.equal((await call("PUT", "/close", { alertId: a.id })).status, 200); const incidentId = (await openIncidentIds([a.id])).get(a.id)!; assert.deepEqual((await getIncident(incidentId))!.notes.map((n) => n.body), ["Closed without a note."]); await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`); const actions = (await db.select().from(auditLog)).map((e) => `${e.category}/${e.action}`); assert.ok(actions.includes("alert/close") && actions.includes("alert/reopen")); const titles = (await getChangeLog({ days: 1, limit: 300 })).entries.map((e) => e.title); assert.ok(titles.some((t) => /^Alert closed: jobA/.test(t)), "closed"); assert.ok(titles.some((t) => /^Closed alert opened again: jobA/.test(t)), "opened again"); }); test("an alert that is not there is not found, a note that is too long is refused, a viewer cannot close or open", async () => { assert.equal((await call("PUT", "/close", { alertId: "monitoring:nothing:1" })).status, 404); const a = await idOfHeartbeat("jobA"); const long = await call("PUT", "/close", { alertId: a.id, note: "x".repeat(501) }); assert.equal(long.status, 400); who = "viewer"; assert.equal((await call("PUT", "/close", { alertId: a.id })).status, 403); assert.equal((await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`)).status, 403); who = "operator"; assert.equal((await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`)).status, 404, "nothing to open"); }); test("closing an alert that is snoozed makes it closed, and snoozing a closed one makes it a snooze again", async () => { const a = await idOfHeartbeat("jobA"); const until = new Date(Date.now() + 3_600_000).toISOString(); assert.equal((await call("PUT", "/snooze", { alertId: a.id, until })).status, 200); assert.equal((await call("PUT", "/close", { alertId: a.id })).status, 200); let [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id)); assert.deepEqual([row.kind, row.until], ["closed", CLOSED_UNTIL]); assert.equal((await call("PUT", "/snooze", { alertId: a.id, until })).status, 200); [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id)); assert.deepEqual([row.kind, row.severity, row.note], ["snooze", null, null]); await call("DELETE", `/snooze?alertId=${encodeURIComponent(a.id)}`); }); test("a bulk snooze over a closed alert can be undone back to closed, with its note", async () => { const a = await idOfHeartbeat("jobA"); await call("PUT", "/close", { alertId: a.id, note: "accepted" }); const r = await call("POST", "/bulk", { action: "snooze", alertIds: [a.id], until: new Date(Date.now() + 3_600_000).toISOString() }); assert.equal(r.body.changed, 1); assert.equal((await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id)))[0].kind, "snooze"); await call("POST", "/bulk", r.body.undo); const [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id)); assert.deepEqual([row.kind, row.note, row.until], ["closed", "accepted", CLOSED_UNTIL]); await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`); }); }); describe("until it gets worse", () => { const present = (id: string, severity: string, message = "disk is 91% full") => [{ id, message, severity, key: `k:${id}` }]; const NOW = new Date(); test("the same alert stays closed however its message changes, and a milder severity does not open it", async () => { await db.delete(alertSnoozes); await setClosed({ alertId: "x1", conditionKey: "k:x1", message: "disk is 91% full", severity: "warning", note: null, actor: "Olga" }, NOW); assert.deepEqual(await settleSnoozes(present("x1", "warning", "disk is 94% full"), NOW), []); assert.deepEqual(await settleSnoozes(present("x1", "info", "disk is 80% full"), NOW), []); assert.equal((await db.select().from(alertSnoozes)).length, 1, "still closed"); assert.equal((await loadSnoozes(NOW)).get("x1")?.closed, true); }); test("a higher severity opens it again and says so, once", async () => { const reminders = await settleSnoozes(present("x1", "critical", "disk is 97% full"), NOW); assert.deepEqual(reminders.map((r) => [r.key, r.reason, r.severity, r.message]), [["k:x1", "worse", "critical", "disk is 97% full"]]); assert.equal((await db.select().from(alertSnoozes)).length, 0); assert.deepEqual(await settleSnoozes(present("x1", "critical"), NOW), [], "nothing more to say"); }); test("gone for less than half an hour it stays closed; gone for longer it is forgotten, so the same problem later is new", async () => { await db.delete(alertSnoozes); await setClosed({ alertId: "x2", conditionKey: "k:x2", message: "m", severity: "warning", note: null, actor: "Olga" }, NOW); await settleSnoozes([], new Date(NOW.getTime() + GONE_AFTER_MS - 60_000)); assert.equal((await db.select().from(alertSnoozes)).length, 1); await settleSnoozes([], new Date(NOW.getTime() + GONE_AFTER_MS + 60_000)); assert.equal((await db.select().from(alertSnoozes)).length, 0); }); test("a closed alert never expires like a snooze does, however much time passes while it is there", async () => { await db.delete(alertSnoozes); await setClosed({ alertId: "x3", conditionKey: "k:x3", message: "m", severity: "warning", note: null, actor: "Olga" }, NOW); const tenYears = new Date(NOW.getTime() + 10 * 365 * 86_400_000); assert.deepEqual(await settleSnoozes(present("x3", "warning"), tenYears), []); assert.equal((await loadSnoozes(tenYears)).has("x3"), true); }); test("messages about a closed alert are held back like a snoozed one's, and a snooze that ends still reminds as before", async () => { await db.delete(alertSnoozes); await setClosed({ alertId: "x4", conditionKey: "k:x4", message: "m", severity: "warning", note: null, actor: "Olga" }, NOW); await setSnooze({ alertId: "x5", conditionKey: "k:x5", message: "snoozed one", until: new Date(NOW.getTime() + 120_000), actor: "Olga" }, NOW); assert.deepEqual((await withoutSnoozed([{ key: "k:x4" }, { key: "k:other" }, { key: "k:x5" }], NOW)).map((i) => i.key), ["k:other"]); const reminders = await settleSnoozes(present("x5", "warning", "snoozed one"), new Date(NOW.getTime() + 180_000)); assert.deepEqual(reminders.map((r) => [r.key, r.reason]), [["k:x5", "ended"]]); }); });