Say who was told in an alert's timeline

Notifications are written down with the alerts they were about, and the
incident timeline lists each one: which channels took it, which refused and
what they said, or why it was held or not sent. New alerts, recovery notices,
reminders and escalations are all covered.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-11-10 21:14:58 +01:00
1 parent f546971000
commit 461219844b
13 files changed
+9443 -18

No files matched your search

+1 -1
View File
@@ -89,7 +89,7 @@ export async function runEscalation(now: Date = new Date()): Promise<number> {
const lines = items.map((d) => `${d.alert.severity === "critical" ? "✕" : "⚠"} ${d.alert.message}`);
const message = `Nobody has acknowledged ${items.length === 1 ? "this alert" : `these ${items.length} alerts`} after ${words(def.afterMinutes)}:\n\n${lines.join("\n")}\n\nAcknowledge ${items.length === 1 ? "it" : "them"} on the Alerts page to stop the reminders.`;
try {
await notifyAudience("Homelab Manager — Still not acknowledged", message, { channels: def.channels, pushIds: null, ...(def.extraEmail ? { email: def.extraEmail } : {}) }, { quietHours: !def.evenInQuietHours, actions: await actionsFor(items.map((d) => keyOfAlertId(d.alert.id, categoryOf.get(d.alert.id) ?? ""))) });
await notifyAudience("Homelab Manager — Still not acknowledged", message, { channels: def.channels, pushIds: null, ...(def.extraEmail ? { email: def.extraEmail } : {}) }, { quietHours: !def.evenInQuietHours, keys: items.map((d) => d.alert.id), actions: await actionsFor(items.map((d) => keyOfAlertId(d.alert.id, categoryOf.get(d.alert.id) ?? ""))) });
} catch (err) {
console.error("[escalation] couldn't send step", step, err);
continue; // not marked as sent: the next look tries again
+32 -1
View File
@@ -16,6 +16,7 @@ import type { Alert } from "./alertTypes.js";
import { describeAudit, auditTimeToIso, type ActivityTone } from "./serverActivity.js";
import { startupGraceRemainingMs } from "./healthMonitor.js";
import { loadAck, type AckInfo } from "./alertAcks.js";
import { loggedForAlert } from "./notificationLog.js";
export type IncidentRow = typeof alertIncidents.$inferSelect;
@@ -177,7 +178,7 @@ export async function openIncidentIds(alertKeys: string[]): Promise<Map<string,
return new Map(rows.map((r) => [r.key, r.id]));
}
export type TimelineKind = "alert" | "ack" | "maintenance" | "change" | "health" | "run" | "related" | "note";
export type TimelineKind = "alert" | "ack" | "maintenance" | "change" | "health" | "run" | "related" | "note" | "notify";
export interface TimelineEvent {
/** ISO. */
@@ -280,12 +281,42 @@ export async function getIncident(id: number, now: Date = new Date()): Promise<I
for (const r of related) events.push({ at: r.openedAt, kind: "related", tone: "info", title: `Another alert was raised: ${r.category} (${r.severity})`, detail: r.message, actor: null, incidentId: r.id });
}
// Who was told: each notification that was about this alert, and how each channel took it (or why it was held or not sent).
const told = await loggedForAlert(row.alertKey, new Date(from - 5 * 60_000).toISOString(), new Date(to + 30 * 60_000).toISOString());
for (const n of told) events.push({ at: n.at, ...describeNotification(n) });
for (const n of notes) events.push({ at: n.createdAt, kind: "note", tone: "neutral", title: `${n.author} wrote a note`, detail: n.body, actor: n.author, noteId: n.id });
events.sort((a, b) => a.at.localeCompare(b.at));
return { incident, ack: row.resolvedAt ? null : await loadAck(row.alertKey, now), timeline: events, notes: notes.map((n) => ({ id: n.id, author: n.author, body: n.body, at: n.createdAt })) };
}
const CHANNEL_NAMES: Record<string, string> = { gotify: "Gotify", ntfy: "ntfy", telegram: "Telegram", pushover: "Pushover", smtp: "email", webhook: "the webhook", webPush: "push", email: "email (escalation)" };
const list = (names: string[]) => (names.length <= 1 ? (names[0] ?? "") : `${names.slice(0, -1).join(", ")} and ${names[names.length - 1]}`);
/** Pure: how one notification reads in an alert's timeline — who it reached, who it did not and why. The notification's own title says what kind it was (a new alert, a recovery, a reminder). */
export function describeNotification(n: { title: string; status: string; note: string | null; results: { channel: string; ok: boolean; error: string | null }[] }): Pick<TimelineEvent, "kind" | "tone" | "title" | "detail" | "actor"> {
const base = { kind: "notify" as const, actor: null };
const said = n.title.replace(/^Homelab Manager — /, "");
const ok = n.results.filter((r) => r.ok).map((r) => CHANNEL_NAMES[r.channel] ?? r.channel);
const bad = n.results.filter((r) => !r.ok);
const errors = bad.map((r) => `${CHANNEL_NAMES[r.channel] ?? r.channel}: ${r.error ?? "failed"}`).join("; ");
switch (n.status) {
case "sent":
return { ...base, tone: "good", title: `Told through ${list(ok)}`, detail: said };
case "partly":
return { ...base, tone: "bad", title: `Told through ${list(ok)} — ${list(bad.map((r) => CHANNEL_NAMES[r.channel] ?? r.channel))} did not take it`, detail: `${said} · ${errors}` };
case "failed":
return { ...base, tone: "bad", title: "Nobody could be told — every channel failed", detail: `${said} · ${errors}` };
case "held":
return { ...base, tone: "neutral", title: "The message was held back", detail: `${said} · ${n.note ?? "held"}` };
case "muted":
return { ...base, tone: "neutral", title: "Nobody was told — a routing rule says nobody should be", detail: said };
default:
return { ...base, tone: "neutral", title: "Nobody was told — no channel is switched on", detail: said };
}
}
export const MAX_NOTE = 4000;
export async function addNote(incidentId: number, author: string, body: string, now: Date = new Date()): Promise<{ id: number } | null> {
+26 -3
View File
@@ -3,7 +3,7 @@
* for quiet hours, a routing rule said nobody should hear it, or no channel was switched on to send it to. Kept 30 days and at most 1000 entries. Nothing here can make sending fail: writing
* it down is best-effort, like the channels themselves. An error's text is cleaned of every token, key and password in the settings before it is kept, since a channel may say them back.
*/
import { and, desc, eq, like, lt, notInArray, or } from "drizzle-orm";
import { and, desc, eq, gte, like, lt, lte, notInArray, or } from "drizzle-orm";
import { db } from "../db/client.js";
import { notificationLog } from "../db/schema.js";
import { getSettings } from "./settingsStore.js";
@@ -62,13 +62,15 @@ async function trim(now: Date): Promise<void> {
}
/** Writes down one notification: what it said, and how it went (or why it was held or muted). Returns its id, or null when it could not be written — never throws. */
export async function recordNotification(entry: { title: string; message: string; results: ChannelResult[]; status?: LogStatus; note?: string | null; resentOf?: number | null }, now: Date = new Date()): Promise<number | null> {
export const MAX_ALERT_KEYS = 50;
export async function recordNotification(entry: { title: string; message: string; results: ChannelResult[]; status?: LogStatus; note?: string | null; resentOf?: number | null; keys?: string[] }, now: Date = new Date()): Promise<number | null> {
try {
const secrets = await secretsNow();
const results = entry.results.map((r) => ({ channel: r.channel, ok: r.ok, error: r.error === null ? null : clip(scrub(r.error.replace(/\s+/g, " ").trim(), secrets), MAX_ERROR) }));
const [row] = await db
.insert(notificationLog)
.values({ at: now.toISOString(), title: clip(entry.title, 300), message: clip(entry.message, MAX_MESSAGE), status: entry.status ?? statusOf(results), results: JSON.stringify(results), note: entry.note ?? null, resentOf: entry.resentOf ?? null })
.values({ at: now.toISOString(), title: clip(entry.title, 300), message: clip(entry.message, MAX_MESSAGE), status: entry.status ?? statusOf(results), results: JSON.stringify(results), note: entry.note ?? null, alertKeys: entry.keys && entry.keys.length > 0 ? JSON.stringify([...new Set(entry.keys.filter((k) => typeof k === "string" && k !== ""))].slice(0, MAX_ALERT_KEYS)) : null, resentOf: entry.resentOf ?? null })
.returning({ id: notificationLog.id });
await trim(now);
return row.id;
@@ -87,6 +89,27 @@ export function resultsOf(row: Pick<LogRow, "results">): ChannelResult[] {
}
}
/** Pure: the keys a row says it was about. */
export function keysOf(row: Pick<LogRow, "alertKeys">): string[] {
try {
const v = row.alertKeys ? JSON.parse(row.alertKeys) : [];
return Array.isArray(v) ? v.filter((k): k is string => typeof k === "string") : [];
} catch {
return [];
}
}
/** The notifications that were about one alert between two moments, oldest first — what its timeline says about who was told. */
export async function loggedForAlert(alertKey: string, fromIso: string, toIso: string): Promise<(Omit<LogRow, "results"> & { results: ChannelResult[] })[]> {
// The key is looked for as the JSON writes it; a character that LIKE treats as a wildcard can match too much, so what is found is checked again.
const rows = await db
.select()
.from(notificationLog)
.where(and(like(notificationLog.alertKeys, `%${JSON.stringify(alertKey)}%`), gte(notificationLog.at, fromIso), lte(notificationLog.at, toIso)))
.orderBy(notificationLog.at, notificationLog.id);
return rows.filter((r) => keysOf(r).includes(alertKey)).map((r) => ({ ...r, results: resultsOf(r) }));
}
export async function getLogged(id: number): Promise<LogRow | null> {
return (await db.select().from(notificationLog).where(eq(notificationLog.id, id)).limit(1))[0] ?? null;
}
+10 -9
View File
@@ -286,7 +286,7 @@ export async function audienceFor(meta?: NotifyMeta): Promise<Audience> {
}
/** Sends to the channels the audience names (and are switched on), and writes down how each went: the entry's id, or null if it could not be written. */
async function sendToChannels(title: string, message: string, audience: Audience = EVERYONE, actions: NotifyActions | null = null, resentOf: number | null = null): Promise<number | null> {
async function sendToChannels(title: string, message: string, audience: Audience = EVERYONE, actions: NotifyActions | null = null, resentOf: number | null = null, keys: string[] = []): Promise<number | null> {
const to = (c: ChannelId) => audience.channels === null || audience.channels.includes(c);
const jobs: [string, Promise<SendOutcome>][] = [];
if (to("gotify")) jobs.push(["gotify", sendGotify(title, message, actions)]);
@@ -299,30 +299,30 @@ async function sendToChannels(title: string, message: string, audience: Audience
if (audience.email) jobs.push(["email", sendExtraEmail(title, actions ? message + footerText(actions) : message, audience.email)]);
const outcomes = await Promise.all(jobs.map(async ([channel, run]) => [channel, await run] as const));
const results: ChannelResult[] = outcomes.filter(([, o]) => o !== null).map(([channel, o]) => ({ channel, ok: o!.ok, error: o!.ok ? null : o!.error }));
return recordNotification({ title, message, results, resentOf });
return recordNotification({ title, message, results, resentOf, keys });
}
/**
* Sends to an audience that is already worked out: held for the quiet-hours digest if it is quiet (unless told not to), nowhere if a rule
* said nobody. Escalations use this with their own channels.
*/
export async function notifyAudience(title: string, message: string, audience: Audience, options: { quietHours?: boolean; actions?: NotifyActions | null; critical?: boolean } = {}): Promise<void> {
export async function notifyAudience(title: string, message: string, audience: Audience, options: { quietHours?: boolean; actions?: NotifyActions | null; critical?: boolean; /** The alerts it is about (their keys), so each one's timeline can say who was told. */ keys?: string[] } = {}): Promise<void> {
if (isMuted(audience)) {
await recordNotification({ title, message, results: [], status: "muted", note: "A routing rule says nobody should be told." });
await recordNotification({ title, message, results: [], status: "muted", note: "A routing rule says nobody should be told.", keys: options.keys });
return;
}
// Held when quiet hours or an away stretch say so — unless it is critical and they let critical through (services/availability.ts).
const held = options.quietHours !== false ? await holdReason(options.critical === true) : null;
if (held) {
await enqueueNotification(title, message, audience.channels === null && audience.pushIds === null && !audience.email ? null : JSON.stringify(audience));
await recordNotification({ title, message, results: [], status: "held", note: held === "away" ? "Held while you are away; it is sent in the digest when you are back." : "Held for quiet hours; it is sent in the digest when they end." });
await recordNotification({ title, message, results: [], status: "held", note: held === "away" ? "Held while you are away; it is sent in the digest when you are back." : "Held for quiet hours; it is sent in the digest when they end.", keys: options.keys });
return;
}
await sendToChannels(title, message, audience, options.actions ?? null);
await sendToChannels(title, message, audience, options.actions ?? null, null, options.keys ?? []);
}
/** Items that may each be for a different audience (a rule can send a disk warning one place and a server outage another): sent as one message per audience. */
async function notifyGrouped<T>(items: T[], metaOf: (item: T) => NotifyMeta, build: (group: T[]) => { title: string; message: string }, keysOf?: (item: T) => string | undefined): Promise<void> {
async function notifyGrouped<T>(items: T[], metaOf: (item: T) => NotifyMeta, build: (group: T[]) => { title: string; message: string }, keysOf?: (item: T) => string | undefined, options: { actions?: boolean } = {}): Promise<void> {
const groups = new Map<string, { audience: Audience; items: T[] }>();
for (const item of items) {
const audience = await audienceFor(metaOf(item));
@@ -335,7 +335,8 @@ async function notifyGrouped<T>(items: T[], metaOf: (item: T) => NotifyMeta, bui
const { title, message } = build(group.items);
// Buttons, when the message is about alerts that have a key and an administrator has switched them on (a message held for quiet hours is sent without them).
const critical = group.items.some((i) => metaOf(i).severity === "critical");
await notifyAudience(title, message, group.audience, { critical, ...(keysOf ? { actions: await actionsFor(group.items.map(keysOf)) } : {}) });
const keys = keysOf ? group.items.map(keysOf).filter((k): k is string => typeof k === "string") : [];
await notifyAudience(title, message, group.audience, { critical, keys, ...(keysOf && options.actions !== false ? { actions: await actionsFor(group.items.map(keysOf)) } : {}) });
}
}
@@ -657,7 +658,7 @@ export async function notifyHealthRecovered(resolved: { message: string; key?: s
await notifyGrouped(resolved, issueMeta, (group) => ({
title: "Homelab Manager — Health Recovered",
message: `${group.length} issue${group.length !== 1 ? "s" : ""} cleared:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
}));
}), (i) => i.key, { actions: false });
}
/** Alerts that were snoozed, whose time is up and that are still there: said once, as a reminder. */