- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics - Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents, incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs, related alerts and notes; Incidents page and a digest section - Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download, check, upload, delete, and restore staged and applied at start-up before the database opens (pre-restore copy kept); alerts and /metrics for stale or failing backups - Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
327 lines
17 KiB
TypeScript
327 lines
17 KiB
TypeScript
/**
|
|
* Incidents: each time an alert is raised and later goes away is one incident, with the moment it began and the moment it ended — so
|
|
* "what happened on Tuesday night?" has an answer after the alert itself is long gone. Around each one the page puts together everything the
|
|
* app knows about that time: who acknowledged it and why, the maintenance windows, what was changed on the server, the update runs, the
|
|
* other alerts that came with it, and the notes people wrote.
|
|
*
|
|
* The record is made from the alerts as they are collected (and a check every five minutes so it doesn't depend on anybody looking), by
|
|
* comparing what is there now with what was open. An alert that isn't there is only called resolved after it has been missing for a couple
|
|
* of minutes in a row — much longer when a check couldn't run, since "not there" then may only mean "couldn't look" — and never in the first
|
|
* minutes after the app starts, when server alerts are held back on purpose.
|
|
*/
|
|
import { and, asc, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or } from "drizzle-orm";
|
|
import { db } from "../db/client.js";
|
|
import { alertIncidents, auditLog, incidentNotes, maintenanceWindows, updateRuns } from "../db/schema.js";
|
|
import type { Alert } from "./alertTypes.js";
|
|
import { describeAudit, auditTimeToIso, type ActivityTone } from "./serverActivity.js";
|
|
import { startupGraceRemainingMs } from "./healthMonitor.js";
|
|
import { loadAck, type AckInfo } from "./alertAcks.js";
|
|
|
|
export type IncidentRow = typeof alertIncidents.$inferSelect;
|
|
|
|
/** How long an alert must be missing before the incident is called over. */
|
|
export const MISSING_GRACE_MS = 2 * 60_000;
|
|
/** The same, when a check couldn't run on the last look: not there may then only mean couldn't look. */
|
|
export const MISSING_GRACE_BLIND_MS = 30 * 60_000;
|
|
/** Resolved incidents are kept this long. */
|
|
export const KEEP_DAYS = 400;
|
|
/** Other alerts raised this close to an incident, about the same thing, are shown with it. */
|
|
export const RELATED_WINDOW_MS = 15 * 60_000;
|
|
|
|
const RANK: Record<string, number> = { info: 0, warning: 1, critical: 2 };
|
|
|
|
export interface SnapshotInput {
|
|
alerts: Alert[];
|
|
/** The checks that couldn't run on this look. */
|
|
blind: number;
|
|
generatedAt?: string;
|
|
}
|
|
|
|
export interface SnapshotResult {
|
|
opened: number[];
|
|
resolved: number[];
|
|
}
|
|
|
|
/**
|
|
* Brings the incidents in line with the alerts as they are now. Information-only alerts aren't incidents. Safe to call as often as likes:
|
|
* the same look twice changes nothing.
|
|
*/
|
|
export async function recordAlertSnapshot(input: SnapshotInput, now: Date = new Date(), options: { inGrace?: boolean } = {}): Promise<SnapshotResult> {
|
|
const iso = now.toISOString();
|
|
const present = new Map(input.alerts.filter((a) => a.severity !== "info").map((a) => [a.id, a]));
|
|
const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt));
|
|
const openByKey = new Map(open.map((i) => [i.alertKey, i]));
|
|
const result: SnapshotResult = { opened: [], resolved: [] };
|
|
|
|
for (const [key, a] of present) {
|
|
const existing = openByKey.get(key);
|
|
if (existing) {
|
|
const peak = (RANK[a.severity] ?? 0) > (RANK[existing.peakSeverity] ?? 0) ? a.severity : existing.peakSeverity;
|
|
await db
|
|
.update(alertIncidents)
|
|
.set({ severity: a.severity, peakSeverity: peak, message: a.message, causedBy: a.causedBy?.name ?? null, lastSeenAt: iso, missingSince: null })
|
|
.where(eq(alertIncidents.id, existing.id));
|
|
} else {
|
|
const [row] = await db
|
|
.insert(alertIncidents)
|
|
.values({
|
|
alertKey: key,
|
|
severity: a.severity,
|
|
peakSeverity: a.severity,
|
|
category: a.category,
|
|
source: a.source,
|
|
message: a.message,
|
|
link: a.link,
|
|
targetType: a.target?.type ?? null,
|
|
targetId: a.target?.id ?? null,
|
|
targetName: a.target?.name ?? null,
|
|
causedBy: a.causedBy?.name ?? null,
|
|
openedAt: iso,
|
|
lastSeenAt: iso,
|
|
})
|
|
.returning({ id: alertIncidents.id });
|
|
result.opened.push(row.id);
|
|
}
|
|
}
|
|
|
|
// What was open and isn't there now. Nothing is called over while the app is still settling after a start, when server alerts are held back.
|
|
const grace = input.blind > 0 ? MISSING_GRACE_BLIND_MS : MISSING_GRACE_MS;
|
|
for (const inc of open) {
|
|
if (present.has(inc.alertKey)) continue;
|
|
if (options.inGrace) continue;
|
|
if (inc.missingSince === null) {
|
|
await db.update(alertIncidents).set({ missingSince: iso }).where(eq(alertIncidents.id, inc.id));
|
|
} else if (now.getTime() - Date.parse(inc.missingSince) >= grace) {
|
|
// Over since the first look it was missing, not since this one.
|
|
await db.update(alertIncidents).set({ resolvedAt: inc.missingSince, missingSince: null }).where(eq(alertIncidents.id, inc.id));
|
|
result.resolved.push(inc.id);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/** Called by the collection of the alerts: records what it found, and never lets a problem here stop the alerts from being served. */
|
|
export async function recordFromReport(report: { alerts: Alert[]; couldntCheck: unknown[] }, now: Date = new Date()): Promise<void> {
|
|
try {
|
|
await recordAlertSnapshot({ alerts: report.alerts, blind: report.couldntCheck.length }, now, { inGrace: startupGraceRemainingMs(now.getTime()) > 0 });
|
|
} catch (err) {
|
|
console.error("[incidents] couldn't record the alerts:", err);
|
|
}
|
|
}
|
|
|
|
/** Resolved incidents older than the keeping time are removed (with their notes). */
|
|
export async function pruneIncidents(now: Date = new Date()): Promise<number> {
|
|
const cutoff = new Date(now.getTime() - KEEP_DAYS * 86_400_000).toISOString();
|
|
const old = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(and(isNotNull(alertIncidents.resolvedAt), lte(alertIncidents.resolvedAt, cutoff)));
|
|
if (old.length === 0) return 0;
|
|
await db.delete(alertIncidents).where(inArray(alertIncidents.id, old.map((o) => o.id)));
|
|
return old.length;
|
|
}
|
|
|
|
// ─── Reading them ───────────────────────────────────────────────────────────
|
|
|
|
export interface IncidentSummary {
|
|
id: number;
|
|
alertKey: string;
|
|
severity: string;
|
|
peakSeverity: string;
|
|
category: string;
|
|
source: string;
|
|
message: string;
|
|
link: string | null;
|
|
target: { type: string; id: number; name: string } | null;
|
|
causedBy: string | null;
|
|
openedAt: string;
|
|
resolvedAt: string | null;
|
|
/** In milliseconds: until it was resolved, or until now while it is open. */
|
|
durationMs: number;
|
|
open: boolean;
|
|
notes: number;
|
|
}
|
|
|
|
const summarize = (r: IncidentRow, notes: number, now: Date): IncidentSummary => ({
|
|
id: r.id,
|
|
alertKey: r.alertKey,
|
|
severity: r.severity,
|
|
peakSeverity: r.peakSeverity,
|
|
category: r.category,
|
|
source: r.source,
|
|
message: r.message,
|
|
link: r.link,
|
|
target: r.targetType && r.targetId !== null ? { type: r.targetType, id: r.targetId, name: r.targetName ?? `${r.targetType} #${r.targetId}` } : null,
|
|
causedBy: r.causedBy,
|
|
openedAt: r.openedAt,
|
|
resolvedAt: r.resolvedAt,
|
|
durationMs: Math.max(0, (r.resolvedAt ? Date.parse(r.resolvedAt) : now.getTime()) - Date.parse(r.openedAt)),
|
|
open: r.resolvedAt === null,
|
|
notes,
|
|
});
|
|
|
|
/** Open incidents first (newest first), then the resolved ones, newest first. */
|
|
export async function listIncidents(options: { limit?: number; now?: Date } = {}): Promise<IncidentSummary[]> {
|
|
const now = options.now ?? new Date();
|
|
const limit = options.limit ?? 300;
|
|
const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt));
|
|
const closed = await db.select().from(alertIncidents).where(isNotNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt)).limit(limit);
|
|
const rows = [...open, ...closed];
|
|
const noteRows = rows.length > 0 ? await db.select({ incidentId: incidentNotes.incidentId }).from(incidentNotes).where(inArray(incidentNotes.incidentId, rows.map((r) => r.id))) : [];
|
|
const counts = new Map<number, number>();
|
|
for (const n of noteRows) counts.set(n.incidentId, (counts.get(n.incidentId) ?? 0) + 1);
|
|
return rows.map((r) => summarize(r, counts.get(r.id) ?? 0, now));
|
|
}
|
|
|
|
/** The open incident for each alert id, so the Alerts page can link to it. */
|
|
export async function openIncidentIds(alertKeys: string[]): Promise<Map<string, number>> {
|
|
if (alertKeys.length === 0) return new Map();
|
|
const rows = await db.select({ id: alertIncidents.id, key: alertIncidents.alertKey }).from(alertIncidents).where(and(isNull(alertIncidents.resolvedAt), inArray(alertIncidents.alertKey, alertKeys)));
|
|
return new Map(rows.map((r) => [r.key, r.id]));
|
|
}
|
|
|
|
export type TimelineKind = "alert" | "ack" | "maintenance" | "change" | "health" | "run" | "related" | "note";
|
|
|
|
export interface TimelineEvent {
|
|
/** ISO. */
|
|
at: string;
|
|
kind: TimelineKind;
|
|
tone: ActivityTone;
|
|
title: string;
|
|
detail: string | null;
|
|
actor: string | null;
|
|
/** For a related alert or a note: what it points at. */
|
|
incidentId?: number;
|
|
noteId?: number;
|
|
}
|
|
|
|
export interface IncidentDetail {
|
|
incident: IncidentSummary;
|
|
/** The acknowledgement in force now, if the alert is still open. */
|
|
ack: AckInfo | null;
|
|
timeline: TimelineEvent[];
|
|
notes: { id: number; author: string; body: string; at: string }[];
|
|
}
|
|
|
|
function parse(detail: string | null): Record<string, unknown> {
|
|
if (!detail) return {};
|
|
try {
|
|
const v = JSON.parse(detail);
|
|
return v && typeof v === "object" ? (v as Record<string, unknown>) : {};
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
const words = (ms: number) => {
|
|
const m = Math.round(ms / 60_000);
|
|
if (m < 90) return `${Math.max(1, m)} min`;
|
|
const h = ms / 3600_000;
|
|
return h < 48 ? `${Math.round(h * 10) / 10} h` : `${Math.round(h / 24)} days`;
|
|
};
|
|
|
|
/** Everything the app knows about one incident, in the order it happened. */
|
|
export async function getIncident(id: number, now: Date = new Date()): Promise<IncidentDetail | null> {
|
|
const [row] = await db.select().from(alertIncidents).where(eq(alertIncidents.id, id)).limit(1);
|
|
if (!row) return null;
|
|
const notes = await db.select().from(incidentNotes).where(eq(incidentNotes.incidentId, id)).orderBy(asc(incidentNotes.createdAt));
|
|
const incident = summarize(row, notes.length, now);
|
|
|
|
const from = Date.parse(row.openedAt);
|
|
const to = row.resolvedAt ? Date.parse(row.resolvedAt) : now.getTime();
|
|
const fromIso = new Date(from - 30 * 60_000).toISOString();
|
|
const toIso = new Date(to + 15 * 60_000).toISOString();
|
|
const events: TimelineEvent[] = [];
|
|
|
|
events.push({ at: row.openedAt, kind: "alert", tone: row.peakSeverity === "critical" ? "bad" : "info", title: `Alert raised (${row.severity})`, detail: row.message, actor: null });
|
|
if (row.peakSeverity !== row.severity) {
|
|
events.push({ at: row.resolvedAt ?? row.lastSeenAt, kind: "alert", tone: "neutral", title: `At its worst it was ${row.peakSeverity}`, detail: null, actor: null });
|
|
}
|
|
if (row.causedBy) events.push({ at: row.openedAt, kind: "alert", tone: "info", title: `Held back: caused by ${row.causedBy}`, detail: "Folded under the alert for the machine it runs on.", actor: null });
|
|
if (row.resolvedAt) events.push({ at: row.resolvedAt, kind: "alert", tone: "good", title: "Resolved", detail: `It lasted ${words(to - from)}.`, actor: null });
|
|
|
|
// Acknowledgements and hand-overs of this alert, from the audit log.
|
|
const audits = await db.select().from(auditLog).where(and(gte(auditLog.createdAt, fromIso.slice(0, 19).replace("T", " ")), lte(auditLog.createdAt, toIso.slice(0, 19).replace("T", " ")))).orderBy(asc(auditLog.createdAt));
|
|
for (const a of audits) {
|
|
const d = parse(a.detail);
|
|
if (a.category === "alert") {
|
|
if (d.alertId !== row.alertKey) continue;
|
|
const described = describeAudit(a.category, a.action, d);
|
|
events.push({ at: auditTimeToIso(a.createdAt), kind: "ack", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel });
|
|
continue;
|
|
}
|
|
// Things done to, or noticed about, the thing the alert is about.
|
|
if (row.targetType && row.targetId !== null && a.targetType === row.targetType && a.targetId === String(row.targetId) && a.category !== "maintenance") {
|
|
const described = describeAudit(a.category, a.action, d);
|
|
events.push({ at: auditTimeToIso(a.createdAt), kind: described.kind === "health" ? "health" : "change", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel });
|
|
}
|
|
}
|
|
|
|
if (row.targetType && row.targetId !== null) {
|
|
const windows = await db
|
|
.select()
|
|
.from(maintenanceWindows)
|
|
.where(and(eq(maintenanceWindows.targetType, row.targetType as "server"), eq(maintenanceWindows.targetId, row.targetId), lte(maintenanceWindows.startedAt, new Date(to).toISOString()), gte(maintenanceWindows.endsAt, new Date(from).toISOString())));
|
|
for (const w of windows) {
|
|
if (w.endsAt <= w.startedAt) continue; // withdrawn before it began
|
|
events.push({ at: w.startedAt, kind: "maintenance", tone: "info", title: "Maintenance began — alerts silenced", detail: w.reason, actor: w.createdBy });
|
|
if (w.endsAt <= toIso) events.push({ at: w.endsAt, kind: "maintenance", tone: "neutral", title: "Maintenance ended", detail: null, actor: null });
|
|
}
|
|
if (row.targetType === "server") {
|
|
const runs = await db.select().from(updateRuns).where(and(eq(updateRuns.serverId, row.targetId), lte(updateRuns.startedAt, new Date(to).toISOString()), or(isNull(updateRuns.finishedAt), gte(updateRuns.finishedAt, new Date(from).toISOString()))!));
|
|
for (const r of runs) {
|
|
events.push({ at: r.startedAt, kind: "run", tone: "info", title: `Update run started: ${r.templateName}`, detail: `by ${r.startedBy ?? "someone"}`, actor: r.startedBy });
|
|
if (r.finishedAt) events.push({ at: r.finishedAt, kind: "run", tone: r.status === "done" ? "good" : "bad", title: r.status === "done" ? "Update run finished" : r.status === "cancelled" ? "Update run stopped being followed" : "Update run didn't finish", detail: r.message, actor: null });
|
|
}
|
|
}
|
|
|
|
// Other alerts about the same thing that began around the same time.
|
|
const related = await db
|
|
.select()
|
|
.from(alertIncidents)
|
|
.where(and(eq(alertIncidents.targetType, row.targetType), eq(alertIncidents.targetId, row.targetId), ne(alertIncidents.id, row.id), gte(alertIncidents.openedAt, new Date(from - RELATED_WINDOW_MS).toISOString()), lte(alertIncidents.openedAt, new Date(Math.max(to, from) + RELATED_WINDOW_MS).toISOString())));
|
|
for (const r of related) events.push({ at: r.openedAt, kind: "related", tone: "info", title: `Another alert was raised: ${r.category} (${r.severity})`, detail: r.message, actor: null, incidentId: r.id });
|
|
}
|
|
|
|
for (const n of notes) events.push({ at: n.createdAt, kind: "note", tone: "neutral", title: `${n.author} wrote a note`, detail: n.body, actor: n.author, noteId: n.id });
|
|
|
|
events.sort((a, b) => a.at.localeCompare(b.at));
|
|
return { incident, ack: row.resolvedAt ? null : await loadAck(row.alertKey, now), timeline: events, notes: notes.map((n) => ({ id: n.id, author: n.author, body: n.body, at: n.createdAt })) };
|
|
}
|
|
|
|
export const MAX_NOTE = 4000;
|
|
|
|
export async function addNote(incidentId: number, author: string, body: string, now: Date = new Date()): Promise<{ id: number } | null> {
|
|
const [inc] = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(eq(alertIncidents.id, incidentId)).limit(1);
|
|
if (!inc) return null;
|
|
const [row] = await db.insert(incidentNotes).values({ incidentId, author, body: body.trim(), createdAt: now.toISOString() }).returning({ id: incidentNotes.id });
|
|
return row;
|
|
}
|
|
|
|
export async function removeNote(incidentId: number, noteId: number, actor: { name: string; isAdmin: boolean }): Promise<"removed" | "not_found" | "forbidden"> {
|
|
const [n] = await db.select().from(incidentNotes).where(and(eq(incidentNotes.id, noteId), eq(incidentNotes.incidentId, incidentId))).limit(1);
|
|
if (!n) return "not_found";
|
|
if (n.author !== actor.name && !actor.isAdmin) return "forbidden";
|
|
await db.delete(incidentNotes).where(eq(incidentNotes.id, noteId));
|
|
return "removed";
|
|
}
|
|
|
|
let timer: ReturnType<typeof setInterval> | null = null;
|
|
|
|
/** A look at the alerts every five minutes, so incidents are recorded whether or not anybody has the page open. Once a day it tidies up old ones. */
|
|
export function initIncidentRecorder(lookAtAlerts: () => Promise<unknown>): void {
|
|
if (timer) return;
|
|
let lastPrune = 0;
|
|
const tick = async () => {
|
|
try {
|
|
await lookAtAlerts();
|
|
if (Date.now() - lastPrune > 24 * 3600_000) {
|
|
lastPrune = Date.now();
|
|
await pruneIncidents();
|
|
}
|
|
} catch (err) {
|
|
console.error("[incidents] the five-minute look failed:", err);
|
|
}
|
|
};
|
|
setTimeout(() => void tick(), 60_000).unref();
|
|
timer = setInterval(() => void tick(), 5 * 60_000);
|
|
timer.unref();
|
|
}
|