Acknowledge and hand over alerts, incident timelines, and built-in database backups

- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what
  needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics
- Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents,
  incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs,
  related alerts and notes; Incidents page and a digest section
- Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download,
  check, upload, delete, and restore staged and applied at start-up before the database opens
  (pre-restore copy kept); alerts and /metrics for stale or failing backups
- Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 02:21:08 +02:00
1 parent 7a5b623444
commit 2744a84719
41 files changed
+6573 -26

No files matched your search

+326
View File
@@ -0,0 +1,326 @@
/**
* Incidents: each time an alert is raised and later goes away is one incident, with the moment it began and the moment it ended — so
* "what happened on Tuesday night?" has an answer after the alert itself is long gone. Around each one the page puts together everything the
* app knows about that time: who acknowledged it and why, the maintenance windows, what was changed on the server, the update runs, the
* other alerts that came with it, and the notes people wrote.
*
* The record is made from the alerts as they are collected (and a check every five minutes so it doesn't depend on anybody looking), by
* comparing what is there now with what was open. An alert that isn't there is only called resolved after it has been missing for a couple
* of minutes in a row — much longer when a check couldn't run, since "not there" then may only mean "couldn't look" — and never in the first
* minutes after the app starts, when server alerts are held back on purpose.
*/
import { and, asc, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or } from "drizzle-orm";
import { db } from "../db/client.js";
import { alertIncidents, auditLog, incidentNotes, maintenanceWindows, updateRuns } from "../db/schema.js";
import type { Alert } from "./alertTypes.js";
import { describeAudit, auditTimeToIso, type ActivityTone } from "./serverActivity.js";
import { startupGraceRemainingMs } from "./healthMonitor.js";
import { loadAck, type AckInfo } from "./alertAcks.js";
export type IncidentRow = typeof alertIncidents.$inferSelect;
/** How long an alert must be missing before the incident is called over. */
export const MISSING_GRACE_MS = 2 * 60_000;
/** The same, when a check couldn't run on the last look: not there may then only mean couldn't look. */
export const MISSING_GRACE_BLIND_MS = 30 * 60_000;
/** Resolved incidents are kept this long. */
export const KEEP_DAYS = 400;
/** Other alerts raised this close to an incident, about the same thing, are shown with it. */
export const RELATED_WINDOW_MS = 15 * 60_000;
const RANK: Record<string, number> = { info: 0, warning: 1, critical: 2 };
export interface SnapshotInput {
alerts: Alert[];
/** The checks that couldn't run on this look. */
blind: number;
generatedAt?: string;
}
export interface SnapshotResult {
opened: number[];
resolved: number[];
}
/**
* Brings the incidents in line with the alerts as they are now. Information-only alerts aren't incidents. Safe to call as often as likes:
* the same look twice changes nothing.
*/
export async function recordAlertSnapshot(input: SnapshotInput, now: Date = new Date(), options: { inGrace?: boolean } = {}): Promise<SnapshotResult> {
const iso = now.toISOString();
const present = new Map(input.alerts.filter((a) => a.severity !== "info").map((a) => [a.id, a]));
const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt));
const openByKey = new Map(open.map((i) => [i.alertKey, i]));
const result: SnapshotResult = { opened: [], resolved: [] };
for (const [key, a] of present) {
const existing = openByKey.get(key);
if (existing) {
const peak = (RANK[a.severity] ?? 0) > (RANK[existing.peakSeverity] ?? 0) ? a.severity : existing.peakSeverity;
await db
.update(alertIncidents)
.set({ severity: a.severity, peakSeverity: peak, message: a.message, causedBy: a.causedBy?.name ?? null, lastSeenAt: iso, missingSince: null })
.where(eq(alertIncidents.id, existing.id));
} else {
const [row] = await db
.insert(alertIncidents)
.values({
alertKey: key,
severity: a.severity,
peakSeverity: a.severity,
category: a.category,
source: a.source,
message: a.message,
link: a.link,
targetType: a.target?.type ?? null,
targetId: a.target?.id ?? null,
targetName: a.target?.name ?? null,
causedBy: a.causedBy?.name ?? null,
openedAt: iso,
lastSeenAt: iso,
})
.returning({ id: alertIncidents.id });
result.opened.push(row.id);
}
}
// What was open and isn't there now. Nothing is called over while the app is still settling after a start, when server alerts are held back.
const grace = input.blind > 0 ? MISSING_GRACE_BLIND_MS : MISSING_GRACE_MS;
for (const inc of open) {
if (present.has(inc.alertKey)) continue;
if (options.inGrace) continue;
if (inc.missingSince === null) {
await db.update(alertIncidents).set({ missingSince: iso }).where(eq(alertIncidents.id, inc.id));
} else if (now.getTime() - Date.parse(inc.missingSince) >= grace) {
// Over since the first look it was missing, not since this one.
await db.update(alertIncidents).set({ resolvedAt: inc.missingSince, missingSince: null }).where(eq(alertIncidents.id, inc.id));
result.resolved.push(inc.id);
}
}
return result;
}
/** Called by the collection of the alerts: records what it found, and never lets a problem here stop the alerts from being served. */
export async function recordFromReport(report: { alerts: Alert[]; couldntCheck: unknown[] }, now: Date = new Date()): Promise<void> {
try {
await recordAlertSnapshot({ alerts: report.alerts, blind: report.couldntCheck.length }, now, { inGrace: startupGraceRemainingMs(now.getTime()) > 0 });
} catch (err) {
console.error("[incidents] couldn't record the alerts:", err);
}
}
/** Resolved incidents older than the keeping time are removed (with their notes). */
export async function pruneIncidents(now: Date = new Date()): Promise<number> {
const cutoff = new Date(now.getTime() - KEEP_DAYS * 86_400_000).toISOString();
const old = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(and(isNotNull(alertIncidents.resolvedAt), lte(alertIncidents.resolvedAt, cutoff)));
if (old.length === 0) return 0;
await db.delete(alertIncidents).where(inArray(alertIncidents.id, old.map((o) => o.id)));
return old.length;
}
// ─── Reading them ───────────────────────────────────────────────────────────
export interface IncidentSummary {
id: number;
alertKey: string;
severity: string;
peakSeverity: string;
category: string;
source: string;
message: string;
link: string | null;
target: { type: string; id: number; name: string } | null;
causedBy: string | null;
openedAt: string;
resolvedAt: string | null;
/** In milliseconds: until it was resolved, or until now while it is open. */
durationMs: number;
open: boolean;
notes: number;
}
const summarize = (r: IncidentRow, notes: number, now: Date): IncidentSummary => ({
id: r.id,
alertKey: r.alertKey,
severity: r.severity,
peakSeverity: r.peakSeverity,
category: r.category,
source: r.source,
message: r.message,
link: r.link,
target: r.targetType && r.targetId !== null ? { type: r.targetType, id: r.targetId, name: r.targetName ?? `${r.targetType} #${r.targetId}` } : null,
causedBy: r.causedBy,
openedAt: r.openedAt,
resolvedAt: r.resolvedAt,
durationMs: Math.max(0, (r.resolvedAt ? Date.parse(r.resolvedAt) : now.getTime()) - Date.parse(r.openedAt)),
open: r.resolvedAt === null,
notes,
});
/** Open incidents first (newest first), then the resolved ones, newest first. */
export async function listIncidents(options: { limit?: number; now?: Date } = {}): Promise<IncidentSummary[]> {
const now = options.now ?? new Date();
const limit = options.limit ?? 300;
const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt));
const closed = await db.select().from(alertIncidents).where(isNotNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt)).limit(limit);
const rows = [...open, ...closed];
const noteRows = rows.length > 0 ? await db.select({ incidentId: incidentNotes.incidentId }).from(incidentNotes).where(inArray(incidentNotes.incidentId, rows.map((r) => r.id))) : [];
const counts = new Map<number, number>();
for (const n of noteRows) counts.set(n.incidentId, (counts.get(n.incidentId) ?? 0) + 1);
return rows.map((r) => summarize(r, counts.get(r.id) ?? 0, now));
}
/** The open incident for each alert id, so the Alerts page can link to it. */
export async function openIncidentIds(alertKeys: string[]): Promise<Map<string, number>> {
if (alertKeys.length === 0) return new Map();
const rows = await db.select({ id: alertIncidents.id, key: alertIncidents.alertKey }).from(alertIncidents).where(and(isNull(alertIncidents.resolvedAt), inArray(alertIncidents.alertKey, alertKeys)));
return new Map(rows.map((r) => [r.key, r.id]));
}
export type TimelineKind = "alert" | "ack" | "maintenance" | "change" | "health" | "run" | "related" | "note";
export interface TimelineEvent {
/** ISO. */
at: string;
kind: TimelineKind;
tone: ActivityTone;
title: string;
detail: string | null;
actor: string | null;
/** For a related alert or a note: what it points at. */
incidentId?: number;
noteId?: number;
}
export interface IncidentDetail {
incident: IncidentSummary;
/** The acknowledgement in force now, if the alert is still open. */
ack: AckInfo | null;
timeline: TimelineEvent[];
notes: { id: number; author: string; body: string; at: string }[];
}
function parse(detail: string | null): Record<string, unknown> {
if (!detail) return {};
try {
const v = JSON.parse(detail);
return v && typeof v === "object" ? (v as Record<string, unknown>) : {};
} catch {
return {};
}
}
const words = (ms: number) => {
const m = Math.round(ms / 60_000);
if (m < 90) return `${Math.max(1, m)} min`;
const h = ms / 3600_000;
return h < 48 ? `${Math.round(h * 10) / 10} h` : `${Math.round(h / 24)} days`;
};
/** Everything the app knows about one incident, in the order it happened. */
export async function getIncident(id: number, now: Date = new Date()): Promise<IncidentDetail | null> {
const [row] = await db.select().from(alertIncidents).where(eq(alertIncidents.id, id)).limit(1);
if (!row) return null;
const notes = await db.select().from(incidentNotes).where(eq(incidentNotes.incidentId, id)).orderBy(asc(incidentNotes.createdAt));
const incident = summarize(row, notes.length, now);
const from = Date.parse(row.openedAt);
const to = row.resolvedAt ? Date.parse(row.resolvedAt) : now.getTime();
const fromIso = new Date(from - 30 * 60_000).toISOString();
const toIso = new Date(to + 15 * 60_000).toISOString();
const events: TimelineEvent[] = [];
events.push({ at: row.openedAt, kind: "alert", tone: row.peakSeverity === "critical" ? "bad" : "info", title: `Alert raised (${row.severity})`, detail: row.message, actor: null });
if (row.peakSeverity !== row.severity) {
events.push({ at: row.resolvedAt ?? row.lastSeenAt, kind: "alert", tone: "neutral", title: `At its worst it was ${row.peakSeverity}`, detail: null, actor: null });
}
if (row.causedBy) events.push({ at: row.openedAt, kind: "alert", tone: "info", title: `Held back: caused by ${row.causedBy}`, detail: "Folded under the alert for the machine it runs on.", actor: null });
if (row.resolvedAt) events.push({ at: row.resolvedAt, kind: "alert", tone: "good", title: "Resolved", detail: `It lasted ${words(to - from)}.`, actor: null });
// Acknowledgements and hand-overs of this alert, from the audit log.
const audits = await db.select().from(auditLog).where(and(gte(auditLog.createdAt, fromIso.slice(0, 19).replace("T", " ")), lte(auditLog.createdAt, toIso.slice(0, 19).replace("T", " ")))).orderBy(asc(auditLog.createdAt));
for (const a of audits) {
const d = parse(a.detail);
if (a.category === "alert") {
if (d.alertId !== row.alertKey) continue;
const described = describeAudit(a.category, a.action, d);
events.push({ at: auditTimeToIso(a.createdAt), kind: "ack", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel });
continue;
}
// Things done to, or noticed about, the thing the alert is about.
if (row.targetType && row.targetId !== null && a.targetType === row.targetType && a.targetId === String(row.targetId) && a.category !== "maintenance") {
const described = describeAudit(a.category, a.action, d);
events.push({ at: auditTimeToIso(a.createdAt), kind: described.kind === "health" ? "health" : "change", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel });
}
}
if (row.targetType && row.targetId !== null) {
const windows = await db
.select()
.from(maintenanceWindows)
.where(and(eq(maintenanceWindows.targetType, row.targetType as "server"), eq(maintenanceWindows.targetId, row.targetId), lte(maintenanceWindows.startedAt, new Date(to).toISOString()), gte(maintenanceWindows.endsAt, new Date(from).toISOString())));
for (const w of windows) {
if (w.endsAt <= w.startedAt) continue; // withdrawn before it began
events.push({ at: w.startedAt, kind: "maintenance", tone: "info", title: "Maintenance began — alerts silenced", detail: w.reason, actor: w.createdBy });
if (w.endsAt <= toIso) events.push({ at: w.endsAt, kind: "maintenance", tone: "neutral", title: "Maintenance ended", detail: null, actor: null });
}
if (row.targetType === "server") {
const runs = await db.select().from(updateRuns).where(and(eq(updateRuns.serverId, row.targetId), lte(updateRuns.startedAt, new Date(to).toISOString()), or(isNull(updateRuns.finishedAt), gte(updateRuns.finishedAt, new Date(from).toISOString()))!));
for (const r of runs) {
events.push({ at: r.startedAt, kind: "run", tone: "info", title: `Update run started: ${r.templateName}`, detail: `by ${r.startedBy ?? "someone"}`, actor: r.startedBy });
if (r.finishedAt) events.push({ at: r.finishedAt, kind: "run", tone: r.status === "done" ? "good" : "bad", title: r.status === "done" ? "Update run finished" : r.status === "cancelled" ? "Update run stopped being followed" : "Update run didn't finish", detail: r.message, actor: null });
}
}
// Other alerts about the same thing that began around the same time.
const related = await db
.select()
.from(alertIncidents)
.where(and(eq(alertIncidents.targetType, row.targetType), eq(alertIncidents.targetId, row.targetId), ne(alertIncidents.id, row.id), gte(alertIncidents.openedAt, new Date(from - RELATED_WINDOW_MS).toISOString()), lte(alertIncidents.openedAt, new Date(Math.max(to, from) + RELATED_WINDOW_MS).toISOString())));
for (const r of related) events.push({ at: r.openedAt, kind: "related", tone: "info", title: `Another alert was raised: ${r.category} (${r.severity})`, detail: r.message, actor: null, incidentId: r.id });
}
for (const n of notes) events.push({ at: n.createdAt, kind: "note", tone: "neutral", title: `${n.author} wrote a note`, detail: n.body, actor: n.author, noteId: n.id });
events.sort((a, b) => a.at.localeCompare(b.at));
return { incident, ack: row.resolvedAt ? null : await loadAck(row.alertKey, now), timeline: events, notes: notes.map((n) => ({ id: n.id, author: n.author, body: n.body, at: n.createdAt })) };
}
export const MAX_NOTE = 4000;
export async function addNote(incidentId: number, author: string, body: string, now: Date = new Date()): Promise<{ id: number } | null> {
const [inc] = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(eq(alertIncidents.id, incidentId)).limit(1);
if (!inc) return null;
const [row] = await db.insert(incidentNotes).values({ incidentId, author, body: body.trim(), createdAt: now.toISOString() }).returning({ id: incidentNotes.id });
return row;
}
export async function removeNote(incidentId: number, noteId: number, actor: { name: string; isAdmin: boolean }): Promise<"removed" | "not_found" | "forbidden"> {
const [n] = await db.select().from(incidentNotes).where(and(eq(incidentNotes.id, noteId), eq(incidentNotes.incidentId, incidentId))).limit(1);
if (!n) return "not_found";
if (n.author !== actor.name && !actor.isAdmin) return "forbidden";
await db.delete(incidentNotes).where(eq(incidentNotes.id, noteId));
return "removed";
}
let timer: ReturnType<typeof setInterval> | null = null;
/** A look at the alerts every five minutes, so incidents are recorded whether or not anybody has the page open. Once a day it tidies up old ones. */
export function initIncidentRecorder(lookAtAlerts: () => Promise<unknown>): void {
if (timer) return;
let lastPrune = 0;
const tick = async () => {
try {
await lookAtAlerts();
if (Date.now() - lastPrune > 24 * 3600_000) {
lastPrune = Date.now();
await pruneIncidents();
}
} catch (err) {
console.error("[incidents] the five-minute look failed:", err);
}
};
setTimeout(() => void tick(), 60_000).unref();
timer = setInterval(() => void tick(), 5 * 60_000);
timer.unref();
}