Files
Homelab-manager/server/test/ackAndIncidents.test.ts
bobbanandClaude Sonnet 5.5 2744a84719 Acknowledge and hand over alerts, incident timelines, and built-in database backups
- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what
  needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics
- Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents,
  incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs,
  related alerts and notes; Incidents page and a digest section
- Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download,
  check, upload, delete, and restore staged and applied at start-up before the database opens
  (pre-restore copy kept); alerts and /metrics for stale or failing backups
- Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-18 02:21:08 +02:00

454 lines
26 KiB
TypeScript

/**
* Acknowledging and handing over alerts, and the incident record: when an alert began and ended, and what happened around it.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-incidents-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, auditLog, maintenanceWindows, updateRuns, alertAcks, alertIncidents, incidentNotes } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const acks = await import("../src/services/alertAcks.js");
const inc = await import("../src/services/incidents.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { getServerActivity } = await import("../src/services/serverActivity.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { buildDigest } = await import("../src/services/weeklyDigest.js");
const { alertsRouter } = await import("../src/routes/alerts.js");
const { incidentsRouter } = await import("../src/routes/incidents.js");
const { hashToken } = await import("../src/services/tokens.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const T0 = new Date("2026-10-17T12:00:00.000Z");
const at = (minutes: number) => new Date(T0.getTime() + minutes * 60_000);
const alert = (id: string, extra: Record<string, unknown> = {}) =>
({ id, severity: "warning" as const, category: "disk" as const, source: "Server", message: `${id} is a problem`, link: null, silenced: false, target: null, ...extra }) as never;
let adminId = 0;
let operatorId = 0;
let viewerId = 0;
before(async () => {
const rows = await db
.insert(users)
.values([
{ oidcSub: "ada", name: "Ada", role: "admin" },
{ oidcSub: "olle", name: "Olle", role: "operator" },
{ oidcSub: "vera", name: "Vera", role: "viewer" },
])
.returning();
[adminId, operatorId, viewerId] = rows.map((r) => r.id);
});
after(() => {
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
describe("acknowledging an alert", () => {
test("a note, who has it and for how long are kept; a second one replaces the first; taking it back removes it", async () => {
const first = await acks.setAck({ alertId: "disk:server:1:/", note: " looking at it ", assigneeId: operatorId, hours: 4, actor: "Ada" }, T0);
assert.equal(first.before, null);
assert.equal(first.ack.note, "looking at it");
assert.deepEqual(first.ack.assignee, { id: operatorId, name: "Olle" });
assert.equal(first.ack.ackedBy, "Ada");
assert.equal(first.ack.until, at(240).toISOString());
assert.equal(first.ack.active, true);
const second = await acks.setAck({ alertId: "disk:server:1:/", note: "", assigneeId: null, hours: null, actor: "Olle" }, at(10));
assert.equal(second.before!.note, "looking at it");
assert.equal(second.ack.assignee, null);
assert.equal(second.ack.until, null, "until the alert is gone");
assert.equal((await db.select().from(alertAcks)).length, 1, "one per alert");
assert.equal((await acks.clearAck("disk:server:1:/"))!.ackedBy, "Olle");
assert.equal(await acks.clearAck("disk:server:1:/"), null);
assert.equal((await acks.loadAcks()).size, 0);
});
test("only people who can act can be handed an alert, and the note and the time are limited", async () => {
const refused = async (input: Partial<Parameters<typeof acks.setAck>[0]>, code: string) =>
assert.rejects(acks.setAck({ alertId: "x", note: "", assigneeId: null, hours: null, actor: "Ada", ...input }, T0), (e: unknown) => e instanceof acks.AckError && e.code === code, code);
await refused({ assigneeId: viewerId }, "invalid_assignee");
await refused({ assigneeId: 99999 }, "invalid_assignee");
await refused({ note: "x".repeat(acks.MAX_NOTE + 1) }, "note_too_long");
await refused({ hours: 0 }, "invalid_duration");
await refused({ hours: acks.MAX_HOURS + 1 }, "invalid_duration");
await refused({ hours: 1.5 }, "invalid_duration");
assert.deepEqual((await acks.assignableUsers()).map((u) => u.name), ["Ada", "Olle"], "no viewers in the list");
});
test("it runs out when it was meant to, and only then does the alert count as unattended again", async () => {
await acks.setAck({ alertId: "a:1", note: "", assigneeId: null, hours: 2, actor: "Ada" }, T0);
await acks.setAck({ alertId: "a:2", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
const alerts = [
{ id: "a:1", severity: "critical", silenced: false },
{ id: "a:2", severity: "warning", silenced: false },
{ id: "a:3", severity: "critical", silenced: false },
{ id: "a:4", severity: "warning", silenced: true },
] as { id: string; severity: "critical" | "warning" | "info"; silenced: boolean }[];
assert.deepEqual(acks.attentionCounts(alerts, await acks.loadAcks(at(60))), { critical: 1, warning: 0, acknowledged: 2 }, "a:3 is the only one nobody has looked at; a silenced one is neither");
const later = await acks.loadAcks(at(150));
assert.equal(later.get("a:1")!.active, false);
assert.deepEqual(acks.attentionCounts(alerts, later), { critical: 2, warning: 0, acknowledged: 1 }, "a:1 is unattended again; a:2 has no end");
await db.delete(alertAcks);
});
test("it goes when the alert has been gone for half an hour — not for a blip, and not if the alert is back", async () => {
await acks.setAck({ alertId: "gone", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
await acks.setAck({ alertId: "stays", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(10)), 0, "ten minutes missing is a blip");
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(29)), 0);
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(31)), 1);
assert.deepEqual([...(await acks.loadAcks()).keys()], ["stays"]);
// a problem that returns later is a new one, nobody having looked at it
assert.equal((await acks.loadAcks()).has("gone"), false);
await db.delete(alertAcks);
});
});
describe("the incident record", () => {
const look = (alerts: unknown[], minutes: number, options: { blind?: number; inGrace?: boolean } = {}) => inc.recordAlertSnapshot({ alerts: alerts as never, blind: options.blind ?? 0 }, at(minutes), { inGrace: options.inGrace });
const rows = () => db.select().from(alertIncidents).orderBy(alertIncidents.id);
test("an alert that appears begins an incident; the same look again changes nothing; information-only alerts aren't incidents", async () => {
await db.delete(alertIncidents);
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
const r1 = await look([a, alert("info:thing", { severity: "info" })], 0);
assert.equal(r1.opened.length, 1);
assert.deepEqual((await look([a], 1)).opened, []);
const [row] = await rows();
assert.equal(row.alertKey, "disk:server:1:/");
assert.equal(row.openedAt, at(0).toISOString());
assert.equal(row.resolvedAt, null);
assert.equal(row.targetName, "elsa");
assert.equal(row.lastSeenAt, at(1).toISOString());
assert.equal((await rows()).length, 1);
});
test("it keeps the worst it got to and the latest wording", async () => {
await look([alert("disk:server:1:/", { severity: "critical", message: "now 97% full", target: { type: "server", id: 1, name: "elsa" } })], 5);
await look([alert("disk:server:1:/", { severity: "warning", message: "back to 91% full", target: { type: "server", id: 1, name: "elsa" } })], 6);
const [row] = await rows();
assert.equal(row.severity, "warning");
assert.equal(row.peakSeverity, "critical");
assert.equal(row.message, "back to 91% full");
});
test("it is only called over after being missing for a couple of minutes in a row, and then it ended when it was first missing", async () => {
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
await look([], 10);
let [row] = await rows();
assert.equal(row.resolvedAt, null);
assert.equal(row.missingSince, at(10).toISOString());
await look([a], 11); // it was only a blip
[row] = await rows();
assert.equal(row.missingSince, null, "back before it was called over: the count starts again");
assert.equal(row.resolvedAt, null);
await look([], 20);
await look([], 21);
[row] = await rows();
assert.equal(row.resolvedAt, null, "one minute isn't enough");
const done = await look([], 23);
assert.equal(done.resolved.length, 1);
[row] = await rows();
assert.equal(row.resolvedAt, at(20).toISOString(), "it ended when it first wasn't there, not when that was confirmed");
assert.equal(row.missingSince, null);
});
test("when it comes back it is a new incident, and the old one stays as it was", async () => {
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
const r = await look([a], 60);
assert.equal(r.opened.length, 1);
const all = await rows();
assert.equal(all.length, 2);
assert.equal(all[0].resolvedAt, at(20).toISOString());
assert.equal(all[1].resolvedAt, null);
assert.equal(all[1].peakSeverity, "warning");
});
test("after a check couldn't run it takes much longer to call anything over, and never in the first minutes after the app starts", async () => {
await db.delete(alertIncidents);
const a = alert("proxmox-backup:1:pve1", { category: "backup" });
await look([a], 0);
await look([], 5, { blind: 1 });
await look([], 20, { blind: 1 });
assert.equal((await rows())[0].resolvedAt, null, "fifteen minutes missing isn't enough while a check is failing");
assert.equal((await look([], 36, { blind: 1 })).resolved.length, 1);
await db.delete(alertIncidents);
await look([a], 100);
await look([], 101, { inGrace: true });
await look([], 200, { inGrace: true });
const [row] = await rows();
assert.equal(row.resolvedAt, null, "server alerts are held back on purpose for a while after the app starts, so missing then means nothing");
assert.equal(row.missingSince, null);
});
test("it is recorded from the alerts as they are collected, without anyone asking, and the Alerts page links to it", async () => {
await db.delete(alertIncidents);
const [s] = await db.insert(servers).values({ name: "quiet", apiTokenHash: hashToken("q"), apiTokenPrefix: "q", lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString() }).returning();
invalidateAlertsCache();
await getAlerts(true);
let found: typeof alertIncidents.$inferSelect | undefined;
for (let i = 0; i < 40 && !found; i++) {
found = (await rows()).find((r) => r.alertKey === `offline:offline:server:${s.id}`);
if (!found) await new Promise((r) => setTimeout(r, 50));
}
assert.ok(found, "an incident for the offline server");
assert.equal(found.severity, "critical");
assert.deepEqual([found.targetType, found.targetId, found.targetName], ["server", s.id, "quiet"]);
assert.equal((await inc.openIncidentIds([found.alertKey])).get(found.alertKey), found.id);
await db.delete(servers).where(eq(servers.id, s.id));
});
test("resolved ones are kept for a long while and then removed with their notes; open ones never", async () => {
await db.delete(alertIncidents);
const old = new Date(Date.now() - (inc.KEEP_DAYS + 5) * 86_400_000).toISOString();
const [oldRow] = await db.insert(alertIncidents).values({ alertKey: "old", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "old", openedAt: old, resolvedAt: old, lastSeenAt: old }).returning();
await db.insert(incidentNotes).values({ incidentId: oldRow.id, author: "Ada", body: "n", createdAt: old });
const [openRow] = await db.insert(alertIncidents).values({ alertKey: "ancient-open", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "still open", openedAt: old, lastSeenAt: old }).returning();
assert.equal(await inc.pruneIncidents(), 1);
assert.deepEqual((await rows()).map((r) => r.id), [openRow.id]);
assert.equal((await db.select().from(incidentNotes)).length, 0);
});
});
describe("the timeline around an incident", () => {
let serverId = 0;
let incidentId = 0;
before(async () => {
await db.delete(alertIncidents);
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("e"), apiTokenPrefix: "e" }).returning();
serverId = s.id;
const [row] = await db
.insert(alertIncidents)
.values({ alertKey: `offline:offline:server:${serverId}`, severity: "critical", peakSeverity: "critical", category: "offline", source: "Server", message: "elsa hasn't reported for 2 h", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(0).toISOString(), resolvedAt: at(95).toISOString(), lastSeenAt: at(95).toISOString() })
.returning();
incidentId = row.id;
const stamp = (m: number) => at(m).toISOString().slice(0, 19).replace("T", " ");
const audit = (m: number, category: string, action: string, detail: object, actor = "system", target: object = { targetType: "server", targetId: String(serverId) }) =>
db.insert(auditLog).values({ actorLabel: actor, category, action, detail: JSON.stringify(detail), createdAt: stamp(m), ...target });
await audit(1, "server", "went_offline", { message: "elsa hasn't reported for 2 h" });
await audit(5, "alert", "ack", { alertId: row.alertKey, message: "x", note: "power cut in the cellar", assignee: "Olle", hours: 4 }, "Ada");
await audit(6, "alert", "ack", { alertId: "some:other:alert", note: "not this one" }, "Ada");
await audit(30, "server", "update", { name: "elsa" }, "Ada");
await audit(90, "server", "back_online", {});
await audit(40, "server", "update", {}, "Ada", { targetType: "server", targetId: "9999" }); // another server
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "UPS swap", startedAt: at(20).toISOString(), endsAt: at(60).toISOString(), createdBy: "Ada" });
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "withdrawn", startedAt: at(25).toISOString(), endsAt: at(25).toISOString() });
await db.insert(updateRuns).values({ serverId, integrationId: 1, projectId: 1, templateId: 1, templateName: "Apply updates", hostLimit: "elsa", status: "done", message: "No updates are waiting now; no restart is needed.", startedBy: "Ada", startedAt: at(61).toISOString(), finishedAt: at(80).toISOString() });
await db.insert(alertIncidents).values({ alertKey: `disk:server:${serverId}:/`, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "/ is 93% full", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(8).toISOString(), lastSeenAt: at(8).toISOString() });
await db.insert(alertIncidents).values({ alertKey: `disk:server:${serverId}:/old`, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "long ago", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(-600).toISOString(), lastSeenAt: at(-600).toISOString() });
await inc.addNote(incidentId, "Olle", "Confirmed: the building lost power.", at(15));
});
test("it puts everything in the order it happened, and only what belongs to this incident", async () => {
const d = (await inc.getIncident(incidentId, at(100)))!;
assert.equal(d.incident.open, false);
assert.equal(d.incident.durationMs, 95 * 60_000);
const titles = d.timeline.map((e) => `${e.at.slice(11, 16)} ${e.kind}: ${e.title}`);
assert.deepEqual(titles, [
"12:00 alert: Alert raised (critical)",
"12:01 health: Stopped reporting — marked offline",
"12:05 ack: Acknowledged, handed to Olle",
"12:08 related: Another alert was raised: disk (warning)",
"12:15 note: Olle wrote a note",
"12:20 maintenance: Maintenance began — alerts silenced",
"12:30 change: Details changed",
"12:55 maintenance: Maintenance ended".replace("12:55", "13:00"),
"13:01 run: Update run started: Apply updates",
"13:20 run: Update run finished",
"13:30 health: Reporting again",
"13:35 alert: Resolved",
]);
});
test("the details say what was written and by whom", async () => {
const d = (await inc.getIncident(incidentId, at(100)))!;
const ack = d.timeline.find((e) => e.kind === "ack")!;
assert.equal(ack.actor, "Ada");
assert.match(ack.detail!, /power cut in the cellar · for 4 h/);
const note = d.timeline.find((e) => e.kind === "note")!;
assert.equal(note.detail, "Confirmed: the building lost power.");
assert.ok(note.noteId);
const related = d.timeline.find((e) => e.kind === "related")!;
assert.match(related.detail!, /93% full/);
assert.ok(related.incidentId);
assert.equal(d.timeline.find((e) => e.title === "Resolved")!.detail, "It lasted 1.6 h.");
assert.equal(d.ack, null, "no acknowledgement to show on one that is over");
assert.equal(d.notes.length, 1);
assert.equal((await inc.getIncident(999999)), null);
});
test("an open one runs to now and shows the acknowledgement in force", async () => {
const [row] = await db.select().from(alertIncidents).where(eq(alertIncidents.alertKey, `disk:server:${serverId}:/`));
await acks.setAck({ alertId: row.alertKey, note: "freeing space", assigneeId: operatorId, hours: null, actor: "Ada" }, at(9));
const d = (await inc.getIncident(row.id, at(120)))!;
assert.equal(d.incident.open, true);
assert.equal(d.incident.durationMs, 112 * 60_000);
assert.equal(d.ack!.note, "freeing space");
assert.equal(d.ack!.assignee!.name, "Olle");
assert.ok(!d.timeline.some((e) => e.title === "Resolved"));
await db.delete(alertAcks);
});
test("the list has the open ones first, then the rest newest first, with how many notes each has", async () => {
const list = await inc.listIncidents({ now: at(120) });
assert.equal(list[0].open, true);
const firstClosed = list.findIndex((i) => !i.open);
assert.ok(list.slice(0, firstClosed).every((i) => i.open) && list.slice(firstClosed).every((i) => !i.open));
assert.equal(list.find((i) => i.id === incidentId)!.notes, 1);
assert.deepEqual(list.filter((i) => !i.open).map((i) => i.openedAt), list.filter((i) => !i.open).map((i) => i.openedAt).sort().reverse());
});
});
describe("in the weekly digest", () => {
test("the week's incidents are counted, the longest are named, and acknowledged alerts are said to be", async () => {
const digest = await buildDigest(at(120));
assert.match(digest.message, /INCIDENTS/);
assert.match(digest.message, /\d+ incidents? began, 1 ended, \d+ still open/);
assert.match(digest.message, /elsa hasn't reported for 2 h — lasted 1 h 35 min/);
});
});
describe("through the routes", () => {
let app: Server;
let base = "";
let who = "ada";
let serverId = 0;
let alertId = "";
let incidentId = 0;
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
before(async () => {
await db.delete(alertIncidents);
const [s] = await db.insert(servers).values({ name: "zoe", apiTokenHash: hashToken("z"), apiTokenPrefix: "z", lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString() }).returning();
serverId = s.id;
alertId = `offline:offline:server:${serverId}`.replace("offline:offline", "offline:offline");
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/alerts", alertsRouter);
a.use("/api/incidents", incidentsRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
invalidateAlertsCache();
const report = await (await call("GET", "/alerts?refresh=1")).json();
alertId = report.alerts.find((x: { id: string }) => x.id.includes(`:server:${serverId}`)).id;
for (let i = 0; i < 40 && !incidentId; i++) {
incidentId = (await db.select().from(alertIncidents)).find((r) => r.alertKey === alertId)?.id ?? 0;
if (!incidentId) await new Promise((r) => setTimeout(r, 50));
}
});
after(() => app.close());
test("operators acknowledge and hand over; viewers can look but not; an alert that isn't there can't be", async () => {
who = "vera";
assert.equal((await call("PUT", "/alerts/ack", { alertId })).status, 403);
assert.equal((await call("GET", "/alerts/assignees")).status, 403);
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 403);
assert.equal((await call("GET", "/alerts")).status, 200);
who = "olle";
const names = (await (await call("GET", "/alerts/assignees")).json()).users.map((u: { name: string }) => u.name);
assert.deepEqual(names, ["Ada", "Olle"]);
assert.equal((await call("PUT", "/alerts/ack", { alertId: "no:such:alert" })).status, 404);
assert.equal((await call("PUT", "/alerts/ack", { alertId: "" })).status, 400);
assert.equal((await call("PUT", "/alerts/ack", { alertId, hours: 0 })).status, 400);
const bad = await call("PUT", "/alerts/ack", { alertId, assigneeId: viewerId });
assert.equal(bad.status, 400);
assert.equal((await bad.json()).error, "invalid_assignee");
const res = await call("PUT", "/alerts/ack", { alertId, note: "power cut", assigneeId: adminId, hours: 6 });
assert.equal(res.status, 200);
const { ack } = await res.json();
assert.equal(ack.assignee.name, "Ada");
assert.equal(ack.ackedBy, "Olle");
assert.equal(ack.active, true);
});
test("the Alerts page shows it on the alert, leaves it out of what needs attention, and links to the incident", async () => {
invalidateAlertsCache();
const report = await (await call("GET", "/alerts")).json();
const a = report.alerts.find((x: { id: string }) => x.id === alertId);
assert.equal(a.ack.note, "power cut");
assert.equal(a.incidentId, incidentId);
assert.ok(report.counts.acknowledged >= 1);
assert.ok(report.counts.attention.critical < report.counts.critical, "the whole count is still the whole count; attention leaves out the acknowledged");
const metrics = await buildMetrics();
assert.match(metrics, /homelab_alerts_acknowledged [1-9]/);
assert.match(metrics, /homelab_incidents_open [1-9]/);
});
test("it is in the audit log and in the server's history, and taking it back is too", async () => {
const entry = (await db.select().from(auditLog)).filter((e) => e.category === "alert" && e.action === "ack").pop()!;
assert.equal(entry.targetType, "server");
assert.equal(entry.targetId, String(serverId));
assert.equal(JSON.parse(entry.detail!).assignee, "Ada");
const activity = await getServerActivity(serverId);
assert.ok(activity.some((e) => e.title === "Acknowledged, handed to Ada" && /power cut/.test(e.detail ?? "")));
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 204);
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 404);
invalidateAlertsCache();
assert.equal((await (await call("GET", "/alerts")).json()).alerts.find((x: { id: string }) => x.id === alertId).ack, null);
assert.ok((await getServerActivity(serverId)).some((e) => e.title === "Acknowledgement taken back"));
});
test("everyone signed in can read the incidents; operators write notes; a note can be removed by its author or an administrator", async () => {
who = "vera";
const list = await (await call("GET", "/incidents")).json();
assert.ok(list.incidents.some((i: { id: number }) => i.id === incidentId));
assert.equal((await call("GET", `/incidents/${incidentId}`)).status, 200);
assert.equal((await call("GET", "/incidents/999999")).status, 404);
assert.equal((await call("GET", "/incidents/x")).status, 400);
assert.equal((await call("POST", `/incidents/${incidentId}/notes`, { body: "hello" })).status, 403);
who = "olle";
assert.equal((await call("POST", `/incidents/${incidentId}/notes`, { body: " " })).status, 400);
assert.equal((await call("POST", "/incidents/999999/notes", { body: "x" })).status, 404);
const made = await call("POST", `/incidents/${incidentId}/notes`, { body: "It was the UPS." });
assert.equal(made.status, 201);
const noteId = (await made.json()).id;
const detail = await (await call("GET", `/incidents/${incidentId}`)).json();
assert.ok(detail.timeline.some((e: { kind: string; detail: string }) => e.kind === "note" && e.detail === "It was the UPS."));
// another operator can't remove it, its author and an administrator can
await db.insert(users).values({ oidcSub: "olga", name: "Olga", role: "operator" });
who = "olga";
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 403);
who = "olle";
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 204);
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 404);
const again = await (await call("POST", `/incidents/${incidentId}/notes`, { body: "second" })).json();
who = "ada";
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${again.id}`)).status, 204);
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "incident").map((e) => e.action);
assert.ok(actions.includes("note") && actions.includes("note_removed"));
});
});