/** * The alert noise report: which alerts fire most often, how long they last, and which have been flapping — worked out from the incident record (every time an alert * was raised and later went away is one incident, services/incidents.ts). It is for deciding what to quieten: raise a limit, ignore a service, fix the cause. * * An alert is the same alert each time it comes back (the incident record keys on the alert's own id), so "fired 14 times" is the same disk or the same check * coming and going — which is the noise. Alerts that fired only once are left out: one is an event, not noise. */ import { and, gte } from "drizzle-orm"; import { db } from "../db/client.js"; import { alertIncidents } from "../db/schema.js"; import { flappingKeys, FLAP_WINDOW_MS, keyOfAlertId } from "./flapGuard.js"; import { getSettings } from "./settingsStore.js"; const DAY_MS = 86_400_000; export const NOISE_DAYS = [7, 30, 90, 180] as const; const MAX_ROWS = 50; /** The threshold the report uses for "flapped" when flapping is switched off in Settings (so the figure still means something). */ const DEFAULT_FLAP_RAISES = 3; export interface NoiseRow { /** The alert's id, which is what the incident record keys on. */ alertId: string; /** What it said the last time. */ message: string; category: string; source: string; where: string; /** How many times it was raised in the period. */ fired: number; totalMinutes: number; averageMinutes: number; longestMinutes: number; /** The most times it was raised within any one hour. */ mostInAnHour: number; /** Raised enough times within an hour to count as flapping. */ flapped: boolean; /** Flapping right now: its messages are held back. */ flappingNow: boolean; openNow: boolean; lastAt: string; link: string | null; /** What could be done about it, in a sentence. */ hint: string; } export interface NoiseReport { days: number; from: string; to: string; /** Every incident in the period, and how many different alerts they were. */ incidents: number; distinctAlerts: number; /** Alerts that fired more than once, which are what the table lists. */ repeating: number; flapped: number; /** All the time spent with an alert up, in the period, in minutes. */ totalMinutes: number; byCategory: { category: string; incidents: number; repeating: number }[]; rows: NoiseRow[]; } /** Pure: the most raises within any window of `windowMs` (the times are epoch ms). */ export function mostWithin(times: number[], windowMs: number): number { const sorted = [...times].sort((a, b) => a - b); let best = 0; let from = 0; for (let i = 0; i < sorted.length; i++) { while (sorted[i] - sorted[from] >= windowMs) from++; best = Math.max(best, i - from + 1); } return best; } /** What to try, from the kind of alert and the key it has. Plain advice; nothing it says is done for anyone. */ export function hintFor(category: string, alertId: string): string { const key = keyOfAlertId(alertId, category); if (key.startsWith("rule:")) return "A limit of your own (Alert settings → Alert rules): if it keeps coming back, raise the limit or make it hold for longer before it raises."; if (key.startsWith("new-port:")) return "If this port should be listening, mark it as expected on the server's Ports card."; if (key.startsWith("heartbeat")) return "If the job is just sometimes late, give its heartbeat more grace; if it fails, look at what it printed."; if (key.startsWith("endpoint:")) return "Raise “Alert after … failures in a row” on the check, or its timeout or interval — or fix what is making it fail."; if (key.startsWith("endpoint-cert:")) return "Renew the certificate, or lower how early the check warns."; if (key.startsWith("failed-units:")) return "If this service is not important, take it off the server's “Services to watch” on its Host health card; otherwise fix the service."; if (key.startsWith("disk-forecast:") || category === "disk") return "Raise the limit for this disk (a server can have its own, per disk), or leave it alone on the server's page if it is meant to be full."; if (category === "offline") return "If the server is just slow to report, raise “Server offline after” under Settings → Notifications, or give this server its own limit."; if (category === "integration") return "Raise “Alert after … consecutive failures” under Settings → Notifications; the Call log (Integrations) shows what is failing."; if (category === "backup") return "A guest that is backed up late each time may need its “too old” limit raised on its server, or its job moved."; return "Acknowledge it with a note, silence it with maintenance while you work on it, or fix what raises it."; } export async function buildNoiseReport(days: number, now: Date = new Date()): Promise { const span = (NOISE_DAYS as readonly number[]).includes(days) ? days : 30; const from = new Date(now.getTime() - span * DAY_MS); const rows = await db.select().from(alertIncidents).where(and(gte(alertIncidents.openedAt, from.toISOString()))); const threshold = (await getSettings()).healthChecks.flapRaises || DEFAULT_FLAP_RAISES; const flapping = await flappingKeys().catch(() => new Set()); const groups = new Map(); for (const r of rows) groups.set(r.alertKey, [...(groups.get(r.alertKey) ?? []), r]); let totalMinutes = 0; const out: NoiseRow[] = []; const categories = new Map(); for (const [alertId, list] of groups) { const minutes = list.map((r) => Math.max(1, Math.round(((r.resolvedAt ? Date.parse(r.resolvedAt) : now.getTime()) - Date.parse(r.openedAt)) / 60_000))); const sum = minutes.reduce((n, m) => n + m, 0); totalMinutes += sum; const latest = [...list].sort((a, b) => b.openedAt.localeCompare(a.openedAt))[0]; const c = categories.get(latest.category) ?? { incidents: 0, repeating: 0 }; c.incidents += list.length; if (list.length > 1) c.repeating++; categories.set(latest.category, c); if (list.length < 2) continue; const most = mostWithin(list.map((r) => Date.parse(r.openedAt)), FLAP_WINDOW_MS); out.push({ alertId, message: latest.message, category: latest.category, source: latest.source, where: latest.targetName ?? latest.source, fired: list.length, totalMinutes: sum, averageMinutes: Math.round(sum / list.length), longestMinutes: Math.max(...minutes), mostInAnHour: most, flapped: most >= threshold, flappingNow: flapping.has(keyOfAlertId(alertId, latest.category)), openNow: list.some((r) => r.resolvedAt === null), lastAt: latest.openedAt, link: latest.link, hint: hintFor(latest.category, alertId), }); } out.sort((a, b) => b.fired - a.fired || b.totalMinutes - a.totalMinutes || a.message.localeCompare(b.message)); return { days: span, from: from.toISOString(), to: now.toISOString(), incidents: rows.length, distinctAlerts: groups.size, repeating: out.length, flapped: out.filter((r) => r.flapped).length, totalMinutes, byCategory: [...categories].map(([category, v]) => ({ category, ...v })).sort((a, b) => b.incidents - a.incidents || a.category.localeCompare(b.category)), rows: out.slice(0, MAX_ROWS), }; }