/** * Incidents: each time an alert is raised and later goes away is one incident, with the moment it began and the moment it ended — so * "what happened on Tuesday night?" has an answer after the alert itself is long gone. Around each one the page puts together everything the * app knows about that time: who acknowledged it and why, the maintenance windows, what was changed on the server, the update runs, the * other alerts that came with it, and the notes people wrote. * * The record is made from the alerts as they are collected (and a check every five minutes so it doesn't depend on anybody looking), by * comparing what is there now with what was open. An alert that isn't there is only called resolved after it has been missing for a couple * of minutes in a row — much longer when a check couldn't run, since "not there" then may only mean "couldn't look" — and never in the first * minutes after the app starts, when server alerts are held back on purpose. */ import { and, asc, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or } from "drizzle-orm"; import { db } from "../db/client.js"; import { alertIncidents, auditLog, incidentNotes, maintenanceWindows, updateRuns } from "../db/schema.js"; import type { Alert } from "./alertTypes.js"; import { describeAudit, auditTimeToIso, type ActivityTone } from "./serverActivity.js"; import { startupGraceRemainingMs } from "./healthMonitor.js"; import { loadAck, type AckInfo } from "./alertAcks.js"; export type IncidentRow = typeof alertIncidents.$inferSelect; /** How long an alert must be missing before the incident is called over. */ export const MISSING_GRACE_MS = 2 * 60_000; /** The same, when a check couldn't run on the last look: not there may then only mean couldn't look. */ export const MISSING_GRACE_BLIND_MS = 30 * 60_000; /** Resolved incidents are kept this long. */ export const KEEP_DAYS = 400; /** Other alerts raised this close to an incident, about the same thing, are shown with it. */ export const RELATED_WINDOW_MS = 15 * 60_000; const RANK: Record = { info: 0, warning: 1, critical: 2 }; export interface SnapshotInput { alerts: Alert[]; /** The checks that couldn't run on this look. */ blind: number; generatedAt?: string; } export interface SnapshotResult { opened: number[]; resolved: number[]; } /** * Brings the incidents in line with the alerts as they are now. Information-only alerts aren't incidents. Safe to call as often as likes: * the same look twice changes nothing. */ export async function recordAlertSnapshot(input: SnapshotInput, now: Date = new Date(), options: { inGrace?: boolean } = {}): Promise { const iso = now.toISOString(); const present = new Map(input.alerts.filter((a) => a.severity !== "info").map((a) => [a.id, a])); const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt)); const openByKey = new Map(open.map((i) => [i.alertKey, i])); const result: SnapshotResult = { opened: [], resolved: [] }; for (const [key, a] of present) { const existing = openByKey.get(key); if (existing) { const peak = (RANK[a.severity] ?? 0) > (RANK[existing.peakSeverity] ?? 0) ? a.severity : existing.peakSeverity; await db .update(alertIncidents) .set({ severity: a.severity, peakSeverity: peak, message: a.message, causedBy: a.causedBy?.name ?? null, lastSeenAt: iso, missingSince: null }) .where(eq(alertIncidents.id, existing.id)); } else { const [row] = await db .insert(alertIncidents) .values({ alertKey: key, severity: a.severity, peakSeverity: a.severity, category: a.category, source: a.source, message: a.message, link: a.link, targetType: a.target?.type ?? null, targetId: a.target?.id ?? null, targetName: a.target?.name ?? null, causedBy: a.causedBy?.name ?? null, openedAt: iso, lastSeenAt: iso, }) .returning({ id: alertIncidents.id }); result.opened.push(row.id); } } // What was open and isn't there now. Nothing is called over while the app is still settling after a start, when server alerts are held back. const grace = input.blind > 0 ? MISSING_GRACE_BLIND_MS : MISSING_GRACE_MS; for (const inc of open) { if (present.has(inc.alertKey)) continue; if (options.inGrace) continue; if (inc.missingSince === null) { await db.update(alertIncidents).set({ missingSince: iso }).where(eq(alertIncidents.id, inc.id)); } else if (now.getTime() - Date.parse(inc.missingSince) >= grace) { // Over since the first look it was missing, not since this one. await db.update(alertIncidents).set({ resolvedAt: inc.missingSince, missingSince: null }).where(eq(alertIncidents.id, inc.id)); result.resolved.push(inc.id); } } return result; } /** Called by the collection of the alerts: records what it found, and never lets a problem here stop the alerts from being served. */ export async function recordFromReport(report: { alerts: Alert[]; couldntCheck: unknown[] }, now: Date = new Date()): Promise { try { await recordAlertSnapshot({ alerts: report.alerts, blind: report.couldntCheck.length }, now, { inGrace: startupGraceRemainingMs(now.getTime()) > 0 }); } catch (err) { console.error("[incidents] couldn't record the alerts:", err); } } /** Resolved incidents older than the keeping time are removed (with their notes). */ export async function pruneIncidents(now: Date = new Date()): Promise { const cutoff = new Date(now.getTime() - KEEP_DAYS * 86_400_000).toISOString(); const old = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(and(isNotNull(alertIncidents.resolvedAt), lte(alertIncidents.resolvedAt, cutoff))); if (old.length === 0) return 0; await db.delete(alertIncidents).where(inArray(alertIncidents.id, old.map((o) => o.id))); return old.length; } // ─── Reading them ─────────────────────────────────────────────────────────── export interface IncidentSummary { id: number; alertKey: string; severity: string; peakSeverity: string; category: string; source: string; message: string; link: string | null; target: { type: string; id: number; name: string } | null; causedBy: string | null; openedAt: string; resolvedAt: string | null; /** In milliseconds: until it was resolved, or until now while it is open. */ durationMs: number; open: boolean; notes: number; } const summarize = (r: IncidentRow, notes: number, now: Date): IncidentSummary => ({ id: r.id, alertKey: r.alertKey, severity: r.severity, peakSeverity: r.peakSeverity, category: r.category, source: r.source, message: r.message, link: r.link, target: r.targetType && r.targetId !== null ? { type: r.targetType, id: r.targetId, name: r.targetName ?? `${r.targetType} #${r.targetId}` } : null, causedBy: r.causedBy, openedAt: r.openedAt, resolvedAt: r.resolvedAt, durationMs: Math.max(0, (r.resolvedAt ? Date.parse(r.resolvedAt) : now.getTime()) - Date.parse(r.openedAt)), open: r.resolvedAt === null, notes, }); /** Open incidents first (newest first), then the resolved ones, newest first. */ export async function listIncidents(options: { limit?: number; now?: Date } = {}): Promise { const now = options.now ?? new Date(); const limit = options.limit ?? 300; const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt)); const closed = await db.select().from(alertIncidents).where(isNotNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt)).limit(limit); const rows = [...open, ...closed]; const noteRows = rows.length > 0 ? await db.select({ incidentId: incidentNotes.incidentId }).from(incidentNotes).where(inArray(incidentNotes.incidentId, rows.map((r) => r.id))) : []; const counts = new Map(); for (const n of noteRows) counts.set(n.incidentId, (counts.get(n.incidentId) ?? 0) + 1); return rows.map((r) => summarize(r, counts.get(r.id) ?? 0, now)); } /** The open incident for each alert id, so the Alerts page can link to it. */ export async function openIncidentIds(alertKeys: string[]): Promise> { if (alertKeys.length === 0) return new Map(); const rows = await db.select({ id: alertIncidents.id, key: alertIncidents.alertKey }).from(alertIncidents).where(and(isNull(alertIncidents.resolvedAt), inArray(alertIncidents.alertKey, alertKeys))); return new Map(rows.map((r) => [r.key, r.id])); } export type TimelineKind = "alert" | "ack" | "maintenance" | "change" | "health" | "run" | "related" | "note"; export interface TimelineEvent { /** ISO. */ at: string; kind: TimelineKind; tone: ActivityTone; title: string; detail: string | null; actor: string | null; /** For a related alert or a note: what it points at. */ incidentId?: number; noteId?: number; } export interface IncidentDetail { incident: IncidentSummary; /** The acknowledgement in force now, if the alert is still open. */ ack: AckInfo | null; timeline: TimelineEvent[]; notes: { id: number; author: string; body: string; at: string }[]; } function parse(detail: string | null): Record { if (!detail) return {}; try { const v = JSON.parse(detail); return v && typeof v === "object" ? (v as Record) : {}; } catch { return {}; } } const words = (ms: number) => { const m = Math.round(ms / 60_000); if (m < 90) return `${Math.max(1, m)} min`; const h = ms / 3600_000; return h < 48 ? `${Math.round(h * 10) / 10} h` : `${Math.round(h / 24)} days`; }; /** Everything the app knows about one incident, in the order it happened. */ export async function getIncident(id: number, now: Date = new Date()): Promise { const [row] = await db.select().from(alertIncidents).where(eq(alertIncidents.id, id)).limit(1); if (!row) return null; const notes = await db.select().from(incidentNotes).where(eq(incidentNotes.incidentId, id)).orderBy(asc(incidentNotes.createdAt)); const incident = summarize(row, notes.length, now); const from = Date.parse(row.openedAt); const to = row.resolvedAt ? Date.parse(row.resolvedAt) : now.getTime(); const fromIso = new Date(from - 30 * 60_000).toISOString(); const toIso = new Date(to + 15 * 60_000).toISOString(); const events: TimelineEvent[] = []; events.push({ at: row.openedAt, kind: "alert", tone: row.peakSeverity === "critical" ? "bad" : "info", title: `Alert raised (${row.severity})`, detail: row.message, actor: null }); if (row.peakSeverity !== row.severity) { events.push({ at: row.resolvedAt ?? row.lastSeenAt, kind: "alert", tone: "neutral", title: `At its worst it was ${row.peakSeverity}`, detail: null, actor: null }); } if (row.causedBy) events.push({ at: row.openedAt, kind: "alert", tone: "info", title: `Held back: caused by ${row.causedBy}`, detail: "Folded under the alert for the machine it runs on.", actor: null }); if (row.resolvedAt) events.push({ at: row.resolvedAt, kind: "alert", tone: "good", title: "Resolved", detail: `It lasted ${words(to - from)}.`, actor: null }); // Acknowledgements and hand-overs of this alert, from the audit log. const audits = await db.select().from(auditLog).where(and(gte(auditLog.createdAt, fromIso.slice(0, 19).replace("T", " ")), lte(auditLog.createdAt, toIso.slice(0, 19).replace("T", " ")))).orderBy(asc(auditLog.createdAt)); for (const a of audits) { const d = parse(a.detail); if (a.category === "alert") { if (d.alertId !== row.alertKey) continue; const described = describeAudit(a.category, a.action, d); events.push({ at: auditTimeToIso(a.createdAt), kind: "ack", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel }); continue; } // Things done to, or noticed about, the thing the alert is about. if (row.targetType && row.targetId !== null && a.targetType === row.targetType && a.targetId === String(row.targetId) && a.category !== "maintenance") { const described = describeAudit(a.category, a.action, d); events.push({ at: auditTimeToIso(a.createdAt), kind: described.kind === "health" ? "health" : "change", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel }); } } if (row.targetType && row.targetId !== null) { const windows = await db .select() .from(maintenanceWindows) .where(and(eq(maintenanceWindows.targetType, row.targetType as "server"), eq(maintenanceWindows.targetId, row.targetId), lte(maintenanceWindows.startedAt, new Date(to).toISOString()), gte(maintenanceWindows.endsAt, new Date(from).toISOString()))); for (const w of windows) { if (w.endsAt <= w.startedAt) continue; // withdrawn before it began events.push({ at: w.startedAt, kind: "maintenance", tone: "info", title: "Maintenance began — alerts silenced", detail: w.reason, actor: w.createdBy }); if (w.endsAt <= toIso) events.push({ at: w.endsAt, kind: "maintenance", tone: "neutral", title: "Maintenance ended", detail: null, actor: null }); } if (row.targetType === "server") { const runs = await db.select().from(updateRuns).where(and(eq(updateRuns.serverId, row.targetId), lte(updateRuns.startedAt, new Date(to).toISOString()), or(isNull(updateRuns.finishedAt), gte(updateRuns.finishedAt, new Date(from).toISOString()))!)); for (const r of runs) { events.push({ at: r.startedAt, kind: "run", tone: "info", title: `Update run started: ${r.templateName}`, detail: `by ${r.startedBy ?? "someone"}`, actor: r.startedBy }); if (r.finishedAt) events.push({ at: r.finishedAt, kind: "run", tone: r.status === "done" ? "good" : "bad", title: r.status === "done" ? "Update run finished" : r.status === "cancelled" ? "Update run stopped being followed" : "Update run didn't finish", detail: r.message, actor: null }); } } // Other alerts about the same thing that began around the same time. const related = await db .select() .from(alertIncidents) .where(and(eq(alertIncidents.targetType, row.targetType), eq(alertIncidents.targetId, row.targetId), ne(alertIncidents.id, row.id), gte(alertIncidents.openedAt, new Date(from - RELATED_WINDOW_MS).toISOString()), lte(alertIncidents.openedAt, new Date(Math.max(to, from) + RELATED_WINDOW_MS).toISOString()))); for (const r of related) events.push({ at: r.openedAt, kind: "related", tone: "info", title: `Another alert was raised: ${r.category} (${r.severity})`, detail: r.message, actor: null, incidentId: r.id }); } for (const n of notes) events.push({ at: n.createdAt, kind: "note", tone: "neutral", title: `${n.author} wrote a note`, detail: n.body, actor: n.author, noteId: n.id }); events.sort((a, b) => a.at.localeCompare(b.at)); return { incident, ack: row.resolvedAt ? null : await loadAck(row.alertKey, now), timeline: events, notes: notes.map((n) => ({ id: n.id, author: n.author, body: n.body, at: n.createdAt })) }; } export const MAX_NOTE = 4000; export async function addNote(incidentId: number, author: string, body: string, now: Date = new Date()): Promise<{ id: number } | null> { const [inc] = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(eq(alertIncidents.id, incidentId)).limit(1); if (!inc) return null; const [row] = await db.insert(incidentNotes).values({ incidentId, author, body: body.trim(), createdAt: now.toISOString() }).returning({ id: incidentNotes.id }); return row; } export async function removeNote(incidentId: number, noteId: number, actor: { name: string; isAdmin: boolean }): Promise<"removed" | "not_found" | "forbidden"> { const [n] = await db.select().from(incidentNotes).where(and(eq(incidentNotes.id, noteId), eq(incidentNotes.incidentId, incidentId))).limit(1); if (!n) return "not_found"; if (n.author !== actor.name && !actor.isAdmin) return "forbidden"; await db.delete(incidentNotes).where(eq(incidentNotes.id, noteId)); return "removed"; } let timer: ReturnType | null = null; /** A look at the alerts every five minutes, so incidents are recorded whether or not anybody has the page open. Once a day it tidies up old ones. */ export function initIncidentRecorder(lookAtAlerts: () => Promise): void { if (timer) return; let lastPrune = 0; const tick = async () => { try { await lookAtAlerts(); if (Date.now() - lastPrune > 24 * 3600_000) { lastPrune = Date.now(); await pruneIncidents(); } } catch (err) { console.error("[incidents] the five-minute look failed:", err); } }; setTimeout(() => void tick(), 60_000).unref(); timer = setInterval(() => void tick(), 5 * 60_000); timer.unref(); }