Close an alert: dealt with or accepted, hidden until it gets worse

Close... on the Alerts page hides an alert with no end, with an optional reason
that is also written on its incident. It opens again only if it gets more
serious than when it was closed (said once) or the problem has been gone for
half an hour; Open again brings it back at once.

Built on the snooze table (kind, severity and note columns), so everything that
holds back snoozed alerts holds back closed ones. Audit log, change log and
/api/v1 alerts carry it.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-25 20:55:39 +01:00
1 parent 4ff0b44943
commit a94899865e
24 files changed
+7982 -24

No files matched your search

+206
View File
@@ -0,0 +1,206 @@
/**
* Closing an alert: hidden with no end, until it gets worse or has been gone for half an hour; why it was closed goes on its incident; it can be opened again; and how it sits
* beside snoozing.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-close-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, heartbeats, alertSnoozes, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { alertsRouter } = await import("../src/routes/alerts.js");
const { alertsBulkRouter } = await import("../src/routes/bulk.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { setClosed, setSnooze, settleSnoozes, withoutSnoozed, loadSnoozes, GONE_AFTER_MS, CLOSED_UNTIL } = await import("../src/services/alertSnoozes.js");
const { getIncident, openIncidentIds } = await import("../src/services/incidents.js");
const { getChangeLog } = await import("../src/services/changeLog.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
let app: Server;
let base = "";
let who = "operator";
const call = async (method: string, path: string, body?: unknown) => {
const res = await fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
return { status: res.status, body: (await res.json().catch(() => null)) as any };
};
const idOfHeartbeat = async (name: string) => {
invalidateAlertsCache();
const report = await getAlerts(true);
const a = report.alerts.find((x) => x.message.startsWith(name));
assert.ok(a, `an alert about ${name}`);
return a!;
};
before(async () => {
await db.insert(users).values([
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
const old = "2026-01-01 00:00:00";
for (const name of ["jobA", "jobB"]) await db.insert(heartbeats).values({ name, token: `hb_${name}`.padEnd(24, "x"), periodMinutes: 60, graceMinutes: 5, serverId: null, createdAt: old });
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/alerts/bulk", alertsBulkRouter);
a.use("/api/alerts", alertsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/alerts`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
describe("closing through the route", () => {
test("a closed alert is held back like a snoozed one, with who closed it and why; it is not counted as needing attention", async () => {
const a = await idOfHeartbeat("jobA");
assert.equal(a.silenced, false);
const attention = (c: { critical: number; warning: number; info: number }) => c.critical + c.warning + c.info;
const before = (await getAlerts(false)).counts;
const r = await call("PUT", "/close", { alertId: a.id, note: " Replaced the job; the old one is retired. " });
assert.equal(r.status, 200);
assert.deepEqual([r.body.closed.by, r.body.closed.note], ["Olga", "Replaced the job; the old one is retired."]);
invalidateAlertsCache();
const report = await getAlerts(true);
const after = report.alerts.find((x) => x.id === a.id)!;
assert.deepEqual([after.silenced, after.closed?.by, after.closed?.note, after.snoozed], [true, "Olga", "Replaced the job; the old one is retired.", undefined]);
assert.equal(attention(report.counts), attention(before) - 1, "no longer among those that need a look");
assert.equal(report.counts.silenced, before.silenced + 1, "counted as held back");
});
test("the reason is written on the incident, and opening it again says so there too", async () => {
const a = await idOfHeartbeat("jobA");
const incidentId = (await openIncidentIds([a.id])).get(a.id)!;
assert.ok(incidentId, "an incident is open for it");
let notes = (await getIncident(incidentId))!.notes.map((n) => [n.author, n.body]);
assert.deepEqual(notes, [["Olga", "Closed: Replaced the job; the old one is retired."]]);
assert.equal((await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`)).status, 204);
notes = (await getIncident(incidentId))!.notes.map((n) => [n.author, n.body]);
assert.deepEqual(notes.at(-1), ["Olga", "Opened again."]);
invalidateAlertsCache();
const again = (await getAlerts(true)).alerts.find((x) => x.id === a.id)!;
assert.deepEqual([again.silenced, again.closed], [false, undefined], "shown and counted again");
});
test("it is in the audit log and the change log as sentences; closing without a note says so on the incident", async () => {
const a = await idOfHeartbeat("jobB");
assert.equal((await call("PUT", "/close", { alertId: a.id })).status, 200);
const incidentId = (await openIncidentIds([a.id])).get(a.id)!;
assert.deepEqual((await getIncident(incidentId))!.notes.map((n) => n.body), ["Closed without a note."]);
await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`);
const actions = (await db.select().from(auditLog)).map((e) => `${e.category}/${e.action}`);
assert.ok(actions.includes("alert/close") && actions.includes("alert/reopen"));
const titles = (await getChangeLog({ days: 1, limit: 300 })).entries.map((e) => e.title);
assert.ok(titles.some((t) => /^Alert closed: jobA/.test(t)), "closed");
assert.ok(titles.some((t) => /^Closed alert opened again: jobA/.test(t)), "opened again");
});
test("an alert that is not there is not found, a note that is too long is refused, a viewer cannot close or open", async () => {
assert.equal((await call("PUT", "/close", { alertId: "monitoring:nothing:1" })).status, 404);
const a = await idOfHeartbeat("jobA");
const long = await call("PUT", "/close", { alertId: a.id, note: "x".repeat(501) });
assert.equal(long.status, 400);
who = "viewer";
assert.equal((await call("PUT", "/close", { alertId: a.id })).status, 403);
assert.equal((await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`)).status, 403);
who = "operator";
assert.equal((await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`)).status, 404, "nothing to open");
});
test("closing an alert that is snoozed makes it closed, and snoozing a closed one makes it a snooze again", async () => {
const a = await idOfHeartbeat("jobA");
const until = new Date(Date.now() + 3_600_000).toISOString();
assert.equal((await call("PUT", "/snooze", { alertId: a.id, until })).status, 200);
assert.equal((await call("PUT", "/close", { alertId: a.id })).status, 200);
let [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id));
assert.deepEqual([row.kind, row.until], ["closed", CLOSED_UNTIL]);
assert.equal((await call("PUT", "/snooze", { alertId: a.id, until })).status, 200);
[row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id));
assert.deepEqual([row.kind, row.severity, row.note], ["snooze", null, null]);
await call("DELETE", `/snooze?alertId=${encodeURIComponent(a.id)}`);
});
test("a bulk snooze over a closed alert can be undone back to closed, with its note", async () => {
const a = await idOfHeartbeat("jobA");
await call("PUT", "/close", { alertId: a.id, note: "accepted" });
const r = await call("POST", "/bulk", { action: "snooze", alertIds: [a.id], until: new Date(Date.now() + 3_600_000).toISOString() });
assert.equal(r.body.changed, 1);
assert.equal((await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id)))[0].kind, "snooze");
await call("POST", "/bulk", r.body.undo);
const [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id));
assert.deepEqual([row.kind, row.note, row.until], ["closed", "accepted", CLOSED_UNTIL]);
await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`);
});
});
describe("until it gets worse", () => {
const present = (id: string, severity: string, message = "disk is 91% full") => [{ id, message, severity, key: `k:${id}` }];
const NOW = new Date();
test("the same alert stays closed however its message changes, and a milder severity does not open it", async () => {
await db.delete(alertSnoozes);
await setClosed({ alertId: "x1", conditionKey: "k:x1", message: "disk is 91% full", severity: "warning", note: null, actor: "Olga" }, NOW);
assert.deepEqual(await settleSnoozes(present("x1", "warning", "disk is 94% full"), NOW), []);
assert.deepEqual(await settleSnoozes(present("x1", "info", "disk is 80% full"), NOW), []);
assert.equal((await db.select().from(alertSnoozes)).length, 1, "still closed");
assert.equal((await loadSnoozes(NOW)).get("x1")?.closed, true);
});
test("a higher severity opens it again and says so, once", async () => {
const reminders = await settleSnoozes(present("x1", "critical", "disk is 97% full"), NOW);
assert.deepEqual(reminders.map((r) => [r.key, r.reason, r.severity, r.message]), [["k:x1", "worse", "critical", "disk is 97% full"]]);
assert.equal((await db.select().from(alertSnoozes)).length, 0);
assert.deepEqual(await settleSnoozes(present("x1", "critical"), NOW), [], "nothing more to say");
});
test("gone for less than half an hour it stays closed; gone for longer it is forgotten, so the same problem later is new", async () => {
await db.delete(alertSnoozes);
await setClosed({ alertId: "x2", conditionKey: "k:x2", message: "m", severity: "warning", note: null, actor: "Olga" }, NOW);
await settleSnoozes([], new Date(NOW.getTime() + GONE_AFTER_MS - 60_000));
assert.equal((await db.select().from(alertSnoozes)).length, 1);
await settleSnoozes([], new Date(NOW.getTime() + GONE_AFTER_MS + 60_000));
assert.equal((await db.select().from(alertSnoozes)).length, 0);
});
test("a closed alert never expires like a snooze does, however much time passes while it is there", async () => {
await db.delete(alertSnoozes);
await setClosed({ alertId: "x3", conditionKey: "k:x3", message: "m", severity: "warning", note: null, actor: "Olga" }, NOW);
const tenYears = new Date(NOW.getTime() + 10 * 365 * 86_400_000);
assert.deepEqual(await settleSnoozes(present("x3", "warning"), tenYears), []);
assert.equal((await loadSnoozes(tenYears)).has("x3"), true);
});
test("messages about a closed alert are held back like a snoozed one's, and a snooze that ends still reminds as before", async () => {
await db.delete(alertSnoozes);
await setClosed({ alertId: "x4", conditionKey: "k:x4", message: "m", severity: "warning", note: null, actor: "Olga" }, NOW);
await setSnooze({ alertId: "x5", conditionKey: "k:x5", message: "snoozed one", until: new Date(NOW.getTime() + 120_000), actor: "Olga" }, NOW);
assert.deepEqual((await withoutSnoozed([{ key: "k:x4" }, { key: "k:other" }, { key: "k:x5" }], NOW)).map((i) => i.key), ["k:other"]);
const reminders = await settleSnoozes(present("x5", "warning", "snoozed one"), new Date(NOW.getTime() + 180_000));
assert.deepEqual(reminders.map((r) => [r.key, r.reason]), [["k:x5", "ended"]]);
});
});