Acknowledge and hand over alerts, incident timelines, and built-in database backups
- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics - Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents, incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs, related alerts and notes; Incidents page and a digest section - Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download, check, upload, delete, and restore staged and applied at start-up before the database opens (pre-restore copy kept); alerts and /metrics for stale or failing backups - Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7a5b623444
commit
2744a84719
41 files changed
+6573
-26
No files matched your search
@@ -0,0 +1,453 @@
|
||||
/**
|
||||
* Acknowledging and handing over alerts, and the incident record: when an alert began and ended, and what happened around it.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-incidents-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, auditLog, maintenanceWindows, updateRuns, alertAcks, alertIncidents, incidentNotes } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const acks = await import("../src/services/alertAcks.js");
|
||||
const inc = await import("../src/services/incidents.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { getServerActivity } = await import("../src/services/serverActivity.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { buildDigest } = await import("../src/services/weeklyDigest.js");
|
||||
const { alertsRouter } = await import("../src/routes/alerts.js");
|
||||
const { incidentsRouter } = await import("../src/routes/incidents.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const T0 = new Date("2026-10-17T12:00:00.000Z");
|
||||
const at = (minutes: number) => new Date(T0.getTime() + minutes * 60_000);
|
||||
const alert = (id: string, extra: Record<string, unknown> = {}) =>
|
||||
({ id, severity: "warning" as const, category: "disk" as const, source: "Server", message: `${id} is a problem`, link: null, silenced: false, target: null, ...extra }) as never;
|
||||
|
||||
let adminId = 0;
|
||||
let operatorId = 0;
|
||||
let viewerId = 0;
|
||||
before(async () => {
|
||||
const rows = await db
|
||||
.insert(users)
|
||||
.values([
|
||||
{ oidcSub: "ada", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "olle", name: "Olle", role: "operator" },
|
||||
{ oidcSub: "vera", name: "Vera", role: "viewer" },
|
||||
])
|
||||
.returning();
|
||||
[adminId, operatorId, viewerId] = rows.map((r) => r.id);
|
||||
});
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
describe("acknowledging an alert", () => {
|
||||
test("a note, who has it and for how long are kept; a second one replaces the first; taking it back removes it", async () => {
|
||||
const first = await acks.setAck({ alertId: "disk:server:1:/", note: " looking at it ", assigneeId: operatorId, hours: 4, actor: "Ada" }, T0);
|
||||
assert.equal(first.before, null);
|
||||
assert.equal(first.ack.note, "looking at it");
|
||||
assert.deepEqual(first.ack.assignee, { id: operatorId, name: "Olle" });
|
||||
assert.equal(first.ack.ackedBy, "Ada");
|
||||
assert.equal(first.ack.until, at(240).toISOString());
|
||||
assert.equal(first.ack.active, true);
|
||||
|
||||
const second = await acks.setAck({ alertId: "disk:server:1:/", note: "", assigneeId: null, hours: null, actor: "Olle" }, at(10));
|
||||
assert.equal(second.before!.note, "looking at it");
|
||||
assert.equal(second.ack.assignee, null);
|
||||
assert.equal(second.ack.until, null, "until the alert is gone");
|
||||
assert.equal((await db.select().from(alertAcks)).length, 1, "one per alert");
|
||||
|
||||
assert.equal((await acks.clearAck("disk:server:1:/"))!.ackedBy, "Olle");
|
||||
assert.equal(await acks.clearAck("disk:server:1:/"), null);
|
||||
assert.equal((await acks.loadAcks()).size, 0);
|
||||
});
|
||||
|
||||
test("only people who can act can be handed an alert, and the note and the time are limited", async () => {
|
||||
const refused = async (input: Partial<Parameters<typeof acks.setAck>[0]>, code: string) =>
|
||||
assert.rejects(acks.setAck({ alertId: "x", note: "", assigneeId: null, hours: null, actor: "Ada", ...input }, T0), (e: unknown) => e instanceof acks.AckError && e.code === code, code);
|
||||
await refused({ assigneeId: viewerId }, "invalid_assignee");
|
||||
await refused({ assigneeId: 99999 }, "invalid_assignee");
|
||||
await refused({ note: "x".repeat(acks.MAX_NOTE + 1) }, "note_too_long");
|
||||
await refused({ hours: 0 }, "invalid_duration");
|
||||
await refused({ hours: acks.MAX_HOURS + 1 }, "invalid_duration");
|
||||
await refused({ hours: 1.5 }, "invalid_duration");
|
||||
assert.deepEqual((await acks.assignableUsers()).map((u) => u.name), ["Ada", "Olle"], "no viewers in the list");
|
||||
});
|
||||
|
||||
test("it runs out when it was meant to, and only then does the alert count as unattended again", async () => {
|
||||
await acks.setAck({ alertId: "a:1", note: "", assigneeId: null, hours: 2, actor: "Ada" }, T0);
|
||||
await acks.setAck({ alertId: "a:2", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
|
||||
const alerts = [
|
||||
{ id: "a:1", severity: "critical", silenced: false },
|
||||
{ id: "a:2", severity: "warning", silenced: false },
|
||||
{ id: "a:3", severity: "critical", silenced: false },
|
||||
{ id: "a:4", severity: "warning", silenced: true },
|
||||
] as { id: string; severity: "critical" | "warning" | "info"; silenced: boolean }[];
|
||||
assert.deepEqual(acks.attentionCounts(alerts, await acks.loadAcks(at(60))), { critical: 1, warning: 0, acknowledged: 2 }, "a:3 is the only one nobody has looked at; a silenced one is neither");
|
||||
const later = await acks.loadAcks(at(150));
|
||||
assert.equal(later.get("a:1")!.active, false);
|
||||
assert.deepEqual(acks.attentionCounts(alerts, later), { critical: 2, warning: 0, acknowledged: 1 }, "a:1 is unattended again; a:2 has no end");
|
||||
await db.delete(alertAcks);
|
||||
});
|
||||
|
||||
test("it goes when the alert has been gone for half an hour — not for a blip, and not if the alert is back", async () => {
|
||||
await acks.setAck({ alertId: "gone", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
|
||||
await acks.setAck({ alertId: "stays", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
|
||||
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(10)), 0, "ten minutes missing is a blip");
|
||||
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(29)), 0);
|
||||
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(31)), 1);
|
||||
assert.deepEqual([...(await acks.loadAcks()).keys()], ["stays"]);
|
||||
// a problem that returns later is a new one, nobody having looked at it
|
||||
assert.equal((await acks.loadAcks()).has("gone"), false);
|
||||
await db.delete(alertAcks);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the incident record", () => {
|
||||
const look = (alerts: unknown[], minutes: number, options: { blind?: number; inGrace?: boolean } = {}) => inc.recordAlertSnapshot({ alerts: alerts as never, blind: options.blind ?? 0 }, at(minutes), { inGrace: options.inGrace });
|
||||
const rows = () => db.select().from(alertIncidents).orderBy(alertIncidents.id);
|
||||
|
||||
test("an alert that appears begins an incident; the same look again changes nothing; information-only alerts aren't incidents", async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
|
||||
const r1 = await look([a, alert("info:thing", { severity: "info" })], 0);
|
||||
assert.equal(r1.opened.length, 1);
|
||||
assert.deepEqual((await look([a], 1)).opened, []);
|
||||
const [row] = await rows();
|
||||
assert.equal(row.alertKey, "disk:server:1:/");
|
||||
assert.equal(row.openedAt, at(0).toISOString());
|
||||
assert.equal(row.resolvedAt, null);
|
||||
assert.equal(row.targetName, "elsa");
|
||||
assert.equal(row.lastSeenAt, at(1).toISOString());
|
||||
assert.equal((await rows()).length, 1);
|
||||
});
|
||||
|
||||
test("it keeps the worst it got to and the latest wording", async () => {
|
||||
await look([alert("disk:server:1:/", { severity: "critical", message: "now 97% full", target: { type: "server", id: 1, name: "elsa" } })], 5);
|
||||
await look([alert("disk:server:1:/", { severity: "warning", message: "back to 91% full", target: { type: "server", id: 1, name: "elsa" } })], 6);
|
||||
const [row] = await rows();
|
||||
assert.equal(row.severity, "warning");
|
||||
assert.equal(row.peakSeverity, "critical");
|
||||
assert.equal(row.message, "back to 91% full");
|
||||
});
|
||||
|
||||
test("it is only called over after being missing for a couple of minutes in a row, and then it ended when it was first missing", async () => {
|
||||
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
|
||||
await look([], 10);
|
||||
let [row] = await rows();
|
||||
assert.equal(row.resolvedAt, null);
|
||||
assert.equal(row.missingSince, at(10).toISOString());
|
||||
await look([a], 11); // it was only a blip
|
||||
[row] = await rows();
|
||||
assert.equal(row.missingSince, null, "back before it was called over: the count starts again");
|
||||
assert.equal(row.resolvedAt, null);
|
||||
|
||||
await look([], 20);
|
||||
await look([], 21);
|
||||
[row] = await rows();
|
||||
assert.equal(row.resolvedAt, null, "one minute isn't enough");
|
||||
const done = await look([], 23);
|
||||
assert.equal(done.resolved.length, 1);
|
||||
[row] = await rows();
|
||||
assert.equal(row.resolvedAt, at(20).toISOString(), "it ended when it first wasn't there, not when that was confirmed");
|
||||
assert.equal(row.missingSince, null);
|
||||
});
|
||||
|
||||
test("when it comes back it is a new incident, and the old one stays as it was", async () => {
|
||||
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
|
||||
const r = await look([a], 60);
|
||||
assert.equal(r.opened.length, 1);
|
||||
const all = await rows();
|
||||
assert.equal(all.length, 2);
|
||||
assert.equal(all[0].resolvedAt, at(20).toISOString());
|
||||
assert.equal(all[1].resolvedAt, null);
|
||||
assert.equal(all[1].peakSeverity, "warning");
|
||||
});
|
||||
|
||||
test("after a check couldn't run it takes much longer to call anything over, and never in the first minutes after the app starts", async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const a = alert("proxmox-backup:1:pve1", { category: "backup" });
|
||||
await look([a], 0);
|
||||
await look([], 5, { blind: 1 });
|
||||
await look([], 20, { blind: 1 });
|
||||
assert.equal((await rows())[0].resolvedAt, null, "fifteen minutes missing isn't enough while a check is failing");
|
||||
assert.equal((await look([], 36, { blind: 1 })).resolved.length, 1);
|
||||
|
||||
await db.delete(alertIncidents);
|
||||
await look([a], 100);
|
||||
await look([], 101, { inGrace: true });
|
||||
await look([], 200, { inGrace: true });
|
||||
const [row] = await rows();
|
||||
assert.equal(row.resolvedAt, null, "server alerts are held back on purpose for a while after the app starts, so missing then means nothing");
|
||||
assert.equal(row.missingSince, null);
|
||||
});
|
||||
|
||||
test("it is recorded from the alerts as they are collected, without anyone asking, and the Alerts page links to it", async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const [s] = await db.insert(servers).values({ name: "quiet", apiTokenHash: hashToken("q"), apiTokenPrefix: "q", lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString() }).returning();
|
||||
invalidateAlertsCache();
|
||||
await getAlerts(true);
|
||||
let found: typeof alertIncidents.$inferSelect | undefined;
|
||||
for (let i = 0; i < 40 && !found; i++) {
|
||||
found = (await rows()).find((r) => r.alertKey === `offline:offline:server:${s.id}`);
|
||||
if (!found) await new Promise((r) => setTimeout(r, 50));
|
||||
}
|
||||
assert.ok(found, "an incident for the offline server");
|
||||
assert.equal(found.severity, "critical");
|
||||
assert.deepEqual([found.targetType, found.targetId, found.targetName], ["server", s.id, "quiet"]);
|
||||
assert.equal((await inc.openIncidentIds([found.alertKey])).get(found.alertKey), found.id);
|
||||
await db.delete(servers).where(eq(servers.id, s.id));
|
||||
});
|
||||
|
||||
test("resolved ones are kept for a long while and then removed with their notes; open ones never", async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const old = new Date(Date.now() - (inc.KEEP_DAYS + 5) * 86_400_000).toISOString();
|
||||
const [oldRow] = await db.insert(alertIncidents).values({ alertKey: "old", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "old", openedAt: old, resolvedAt: old, lastSeenAt: old }).returning();
|
||||
await db.insert(incidentNotes).values({ incidentId: oldRow.id, author: "Ada", body: "n", createdAt: old });
|
||||
const [openRow] = await db.insert(alertIncidents).values({ alertKey: "ancient-open", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "still open", openedAt: old, lastSeenAt: old }).returning();
|
||||
assert.equal(await inc.pruneIncidents(), 1);
|
||||
assert.deepEqual((await rows()).map((r) => r.id), [openRow.id]);
|
||||
assert.equal((await db.select().from(incidentNotes)).length, 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the timeline around an incident", () => {
|
||||
let serverId = 0;
|
||||
let incidentId = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("e"), apiTokenPrefix: "e" }).returning();
|
||||
serverId = s.id;
|
||||
const [row] = await db
|
||||
.insert(alertIncidents)
|
||||
.values({ alertKey: `offline:offline:server:${serverId}`, severity: "critical", peakSeverity: "critical", category: "offline", source: "Server", message: "elsa hasn't reported for 2 h", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(0).toISOString(), resolvedAt: at(95).toISOString(), lastSeenAt: at(95).toISOString() })
|
||||
.returning();
|
||||
incidentId = row.id;
|
||||
const stamp = (m: number) => at(m).toISOString().slice(0, 19).replace("T", " ");
|
||||
const audit = (m: number, category: string, action: string, detail: object, actor = "system", target: object = { targetType: "server", targetId: String(serverId) }) =>
|
||||
db.insert(auditLog).values({ actorLabel: actor, category, action, detail: JSON.stringify(detail), createdAt: stamp(m), ...target });
|
||||
await audit(1, "server", "went_offline", { message: "elsa hasn't reported for 2 h" });
|
||||
await audit(5, "alert", "ack", { alertId: row.alertKey, message: "x", note: "power cut in the cellar", assignee: "Olle", hours: 4 }, "Ada");
|
||||
await audit(6, "alert", "ack", { alertId: "some:other:alert", note: "not this one" }, "Ada");
|
||||
await audit(30, "server", "update", { name: "elsa" }, "Ada");
|
||||
await audit(90, "server", "back_online", {});
|
||||
await audit(40, "server", "update", {}, "Ada", { targetType: "server", targetId: "9999" }); // another server
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "UPS swap", startedAt: at(20).toISOString(), endsAt: at(60).toISOString(), createdBy: "Ada" });
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "withdrawn", startedAt: at(25).toISOString(), endsAt: at(25).toISOString() });
|
||||
await db.insert(updateRuns).values({ serverId, integrationId: 1, projectId: 1, templateId: 1, templateName: "Apply updates", hostLimit: "elsa", status: "done", message: "No updates are waiting now; no restart is needed.", startedBy: "Ada", startedAt: at(61).toISOString(), finishedAt: at(80).toISOString() });
|
||||
await db.insert(alertIncidents).values({ alertKey: `disk:server:${serverId}:/`, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "/ is 93% full", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(8).toISOString(), lastSeenAt: at(8).toISOString() });
|
||||
await db.insert(alertIncidents).values({ alertKey: `disk:server:${serverId}:/old`, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "long ago", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(-600).toISOString(), lastSeenAt: at(-600).toISOString() });
|
||||
await inc.addNote(incidentId, "Olle", "Confirmed: the building lost power.", at(15));
|
||||
});
|
||||
|
||||
test("it puts everything in the order it happened, and only what belongs to this incident", async () => {
|
||||
const d = (await inc.getIncident(incidentId, at(100)))!;
|
||||
assert.equal(d.incident.open, false);
|
||||
assert.equal(d.incident.durationMs, 95 * 60_000);
|
||||
const titles = d.timeline.map((e) => `${e.at.slice(11, 16)} ${e.kind}: ${e.title}`);
|
||||
assert.deepEqual(titles, [
|
||||
"12:00 alert: Alert raised (critical)",
|
||||
"12:01 health: Stopped reporting — marked offline",
|
||||
"12:05 ack: Acknowledged, handed to Olle",
|
||||
"12:08 related: Another alert was raised: disk (warning)",
|
||||
"12:15 note: Olle wrote a note",
|
||||
"12:20 maintenance: Maintenance began — alerts silenced",
|
||||
"12:30 change: Details changed",
|
||||
"12:55 maintenance: Maintenance ended".replace("12:55", "13:00"),
|
||||
"13:01 run: Update run started: Apply updates",
|
||||
"13:20 run: Update run finished",
|
||||
"13:30 health: Reporting again",
|
||||
"13:35 alert: Resolved",
|
||||
]);
|
||||
});
|
||||
|
||||
test("the details say what was written and by whom", async () => {
|
||||
const d = (await inc.getIncident(incidentId, at(100)))!;
|
||||
const ack = d.timeline.find((e) => e.kind === "ack")!;
|
||||
assert.equal(ack.actor, "Ada");
|
||||
assert.match(ack.detail!, /power cut in the cellar · for 4 h/);
|
||||
const note = d.timeline.find((e) => e.kind === "note")!;
|
||||
assert.equal(note.detail, "Confirmed: the building lost power.");
|
||||
assert.ok(note.noteId);
|
||||
const related = d.timeline.find((e) => e.kind === "related")!;
|
||||
assert.match(related.detail!, /93% full/);
|
||||
assert.ok(related.incidentId);
|
||||
assert.equal(d.timeline.find((e) => e.title === "Resolved")!.detail, "It lasted 1.6 h.");
|
||||
assert.equal(d.ack, null, "no acknowledgement to show on one that is over");
|
||||
assert.equal(d.notes.length, 1);
|
||||
assert.equal((await inc.getIncident(999999)), null);
|
||||
});
|
||||
|
||||
test("an open one runs to now and shows the acknowledgement in force", async () => {
|
||||
const [row] = await db.select().from(alertIncidents).where(eq(alertIncidents.alertKey, `disk:server:${serverId}:/`));
|
||||
await acks.setAck({ alertId: row.alertKey, note: "freeing space", assigneeId: operatorId, hours: null, actor: "Ada" }, at(9));
|
||||
const d = (await inc.getIncident(row.id, at(120)))!;
|
||||
assert.equal(d.incident.open, true);
|
||||
assert.equal(d.incident.durationMs, 112 * 60_000);
|
||||
assert.equal(d.ack!.note, "freeing space");
|
||||
assert.equal(d.ack!.assignee!.name, "Olle");
|
||||
assert.ok(!d.timeline.some((e) => e.title === "Resolved"));
|
||||
await db.delete(alertAcks);
|
||||
});
|
||||
|
||||
test("the list has the open ones first, then the rest newest first, with how many notes each has", async () => {
|
||||
const list = await inc.listIncidents({ now: at(120) });
|
||||
assert.equal(list[0].open, true);
|
||||
const firstClosed = list.findIndex((i) => !i.open);
|
||||
assert.ok(list.slice(0, firstClosed).every((i) => i.open) && list.slice(firstClosed).every((i) => !i.open));
|
||||
assert.equal(list.find((i) => i.id === incidentId)!.notes, 1);
|
||||
assert.deepEqual(list.filter((i) => !i.open).map((i) => i.openedAt), list.filter((i) => !i.open).map((i) => i.openedAt).sort().reverse());
|
||||
});
|
||||
});
|
||||
|
||||
describe("in the weekly digest", () => {
|
||||
test("the week's incidents are counted, the longest are named, and acknowledged alerts are said to be", async () => {
|
||||
const digest = await buildDigest(at(120));
|
||||
assert.match(digest.message, /INCIDENTS/);
|
||||
assert.match(digest.message, /\d+ incidents? began, 1 ended, \d+ still open/);
|
||||
assert.match(digest.message, /elsa hasn't reported for 2 h — lasted 1 h 35 min/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "ada";
|
||||
let serverId = 0;
|
||||
let alertId = "";
|
||||
let incidentId = 0;
|
||||
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
before(async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const [s] = await db.insert(servers).values({ name: "zoe", apiTokenHash: hashToken("z"), apiTokenPrefix: "z", lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString() }).returning();
|
||||
serverId = s.id;
|
||||
alertId = `offline:offline:server:${serverId}`.replace("offline:offline", "offline:offline");
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/alerts", alertsRouter);
|
||||
a.use("/api/incidents", incidentsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
invalidateAlertsCache();
|
||||
const report = await (await call("GET", "/alerts?refresh=1")).json();
|
||||
alertId = report.alerts.find((x: { id: string }) => x.id.includes(`:server:${serverId}`)).id;
|
||||
for (let i = 0; i < 40 && !incidentId; i++) {
|
||||
incidentId = (await db.select().from(alertIncidents)).find((r) => r.alertKey === alertId)?.id ?? 0;
|
||||
if (!incidentId) await new Promise((r) => setTimeout(r, 50));
|
||||
}
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
test("operators acknowledge and hand over; viewers can look but not; an alert that isn't there can't be", async () => {
|
||||
who = "vera";
|
||||
assert.equal((await call("PUT", "/alerts/ack", { alertId })).status, 403);
|
||||
assert.equal((await call("GET", "/alerts/assignees")).status, 403);
|
||||
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 403);
|
||||
assert.equal((await call("GET", "/alerts")).status, 200);
|
||||
|
||||
who = "olle";
|
||||
const names = (await (await call("GET", "/alerts/assignees")).json()).users.map((u: { name: string }) => u.name);
|
||||
assert.deepEqual(names, ["Ada", "Olle"]);
|
||||
assert.equal((await call("PUT", "/alerts/ack", { alertId: "no:such:alert" })).status, 404);
|
||||
assert.equal((await call("PUT", "/alerts/ack", { alertId: "" })).status, 400);
|
||||
assert.equal((await call("PUT", "/alerts/ack", { alertId, hours: 0 })).status, 400);
|
||||
const bad = await call("PUT", "/alerts/ack", { alertId, assigneeId: viewerId });
|
||||
assert.equal(bad.status, 400);
|
||||
assert.equal((await bad.json()).error, "invalid_assignee");
|
||||
|
||||
const res = await call("PUT", "/alerts/ack", { alertId, note: "power cut", assigneeId: adminId, hours: 6 });
|
||||
assert.equal(res.status, 200);
|
||||
const { ack } = await res.json();
|
||||
assert.equal(ack.assignee.name, "Ada");
|
||||
assert.equal(ack.ackedBy, "Olle");
|
||||
assert.equal(ack.active, true);
|
||||
});
|
||||
|
||||
test("the Alerts page shows it on the alert, leaves it out of what needs attention, and links to the incident", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await (await call("GET", "/alerts")).json();
|
||||
const a = report.alerts.find((x: { id: string }) => x.id === alertId);
|
||||
assert.equal(a.ack.note, "power cut");
|
||||
assert.equal(a.incidentId, incidentId);
|
||||
assert.ok(report.counts.acknowledged >= 1);
|
||||
assert.ok(report.counts.attention.critical < report.counts.critical, "the whole count is still the whole count; attention leaves out the acknowledged");
|
||||
const metrics = await buildMetrics();
|
||||
assert.match(metrics, /homelab_alerts_acknowledged [1-9]/);
|
||||
assert.match(metrics, /homelab_incidents_open [1-9]/);
|
||||
});
|
||||
|
||||
test("it is in the audit log and in the server's history, and taking it back is too", async () => {
|
||||
const entry = (await db.select().from(auditLog)).filter((e) => e.category === "alert" && e.action === "ack").pop()!;
|
||||
assert.equal(entry.targetType, "server");
|
||||
assert.equal(entry.targetId, String(serverId));
|
||||
assert.equal(JSON.parse(entry.detail!).assignee, "Ada");
|
||||
const activity = await getServerActivity(serverId);
|
||||
assert.ok(activity.some((e) => e.title === "Acknowledged, handed to Ada" && /power cut/.test(e.detail ?? "")));
|
||||
|
||||
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 204);
|
||||
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 404);
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await (await call("GET", "/alerts")).json()).alerts.find((x: { id: string }) => x.id === alertId).ack, null);
|
||||
assert.ok((await getServerActivity(serverId)).some((e) => e.title === "Acknowledgement taken back"));
|
||||
});
|
||||
|
||||
test("everyone signed in can read the incidents; operators write notes; a note can be removed by its author or an administrator", async () => {
|
||||
who = "vera";
|
||||
const list = await (await call("GET", "/incidents")).json();
|
||||
assert.ok(list.incidents.some((i: { id: number }) => i.id === incidentId));
|
||||
assert.equal((await call("GET", `/incidents/${incidentId}`)).status, 200);
|
||||
assert.equal((await call("GET", "/incidents/999999")).status, 404);
|
||||
assert.equal((await call("GET", "/incidents/x")).status, 400);
|
||||
assert.equal((await call("POST", `/incidents/${incidentId}/notes`, { body: "hello" })).status, 403);
|
||||
|
||||
who = "olle";
|
||||
assert.equal((await call("POST", `/incidents/${incidentId}/notes`, { body: " " })).status, 400);
|
||||
assert.equal((await call("POST", "/incidents/999999/notes", { body: "x" })).status, 404);
|
||||
const made = await call("POST", `/incidents/${incidentId}/notes`, { body: "It was the UPS." });
|
||||
assert.equal(made.status, 201);
|
||||
const noteId = (await made.json()).id;
|
||||
const detail = await (await call("GET", `/incidents/${incidentId}`)).json();
|
||||
assert.ok(detail.timeline.some((e: { kind: string; detail: string }) => e.kind === "note" && e.detail === "It was the UPS."));
|
||||
|
||||
// another operator can't remove it, its author and an administrator can
|
||||
await db.insert(users).values({ oidcSub: "olga", name: "Olga", role: "operator" });
|
||||
who = "olga";
|
||||
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 403);
|
||||
who = "olle";
|
||||
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 204);
|
||||
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 404);
|
||||
const again = await (await call("POST", `/incidents/${incidentId}/notes`, { body: "second" })).json();
|
||||
who = "ada";
|
||||
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${again.id}`)).status, 204);
|
||||
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "incident").map((e) => e.action);
|
||||
assert.ok(actions.includes("note") && actions.includes("note_removed"));
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user