From 0277126add8d12df27617c034358512a23c5eb82 Mon Sep 17 00:00:00 2001 From: Bobban Rydh Date: Wed, 21 Oct 2026 20:25:45 +0200 Subject: [PATCH] What changed before this? on every incident, ranked - A card on the incident page, and GET /api/incidents/:id/before and /api/v1/incidents/:id/before: what the app knows happened in the 30 min to 24 h before an alert began (audit log, maintenance windows, update runs and patch plans, other alerts, a step in the server's own figures), scored by how related, what kind and how close - A host going down is shown once; Likely, Possible and Background; at most ten - Docs and roles updated Co-Authored-By: Claude Sonnet 5.5 --- README.md | 7 + ROLES.md | 1 + docs/API.md | 28 ++ docs/MONITORING.md | 18 + server/src/routes/incidents.ts | 10 + server/src/routes/publicApi.ts | 10 + server/src/services/incidentBefore.ts | 416 +++++++++++++++++++++++ server/test/incidentBefore.test.ts | 230 +++++++++++++ server/test/readApi.test.ts | 3 + web/src/api/client.ts | 27 ++ web/src/components/WhatChangedBefore.tsx | 135 ++++++++ web/src/pages/IncidentDetail.tsx | 2 + 12 files changed, 887 insertions(+) create mode 100644 server/src/services/incidentBefore.ts create mode 100644 server/test/incidentBefore.test.ts create mode 100644 web/src/components/WhatChangedBefore.tsx diff --git a/README.md b/README.md index e3b3acf..f39897b 100644 --- a/README.md +++ b/README.md @@ -898,6 +898,13 @@ maintenance — it is not counted in the menu badge, the dashboard or the digest promised about the thing. When the time comes an alert that is **still there is announced again, once** ("Snooze ended — an alert you snoozed is still there"); one that went away is simply gone. *Wake now* ends it early. A snooze, like an acknowledgement, is dropped once its alert has been gone for half an hour. The summaries that other checks send (stale backups, expiry, updates) are not held, only what is announced per alert. +**What changed before this?** Every incident page has a card with that name: what the app knows happened in the time before the alert began (30 minutes, 2, 6 or 24 hours), **ranked by how +likely each is to be connected**. It looks through the audit log (settings, DNS, dependencies, a server being changed or going down), maintenance windows that began or ended, update runs and patch plans, other +alerts raised first, and a step in the server's own figures (a disk that jumped from 50% to 73%). Each is scored from how *related* it is (the same thing; something it needs, such as the host it runs on or the +switch it is plugged into; something that needs it; a change that touches everything), what *kind* it is (a maintenance window ending or a host going down is likelier than a runbook being edited) and how +*close* it was. They are shown as *Likely*, *Possible* or *Background* (the last behind a button). It is a list of suspects, not a diagnosis: the app can say what happened and when, not why, and it cannot see a +change made on the machine itself. Everyone signed in can see it; it is also `GET /api/v1/incidents/:id/before` in the read API. + **Ports** — each server's detail page has a Ports card for finding free ports and remembering what each one is for. "Scan…" runs a TCP scan of a port range from the app against the server's address (private addresses only, up to diff --git a/ROLES.md b/ROLES.md index a045760..ed27677 100644 --- a/ROLES.md +++ b/ROLES.md @@ -116,6 +116,7 @@ even further, to admin only: integration itself (a key or login) is an administrator's job, like any integration. - **Alert rules** (Monitoring → Alert rules): everyone signed in sees the rules and where each is raised now; making, changing, trying, switching and removing one is administrator-only (a rule decides who is woken up). **Snooze** an alert, and wake it, need operator, like acknowledging. The storage forecast beside Proxmox storages and Synology volumes is for everyone who can see those pages. +- **What changed before this?** (on an incident's page, and `GET /api/v1/incidents/:id/before`): everyone signed in, like the incident record it is made from. - **Dependencies**: everyone can see what needs what; adding and removing one needs operator. So does adding or hiding a *suggested* one. - **To do and Notes** (Operations → To do, the *To do* and *Notes* cards on a server's page, a device's *Notes* drawer): everyone signed in reads them; adding, editing, ticking off and removing a job, and writing a note, need operator. - **Racks** (Inventory → Racks) and the **network diagram**: everyone signed in. A rack is made from the hardware's *where* and *rack unit*, which operators edit on the inventory page. diff --git a/docs/API.md b/docs/API.md index 098450f..41b903d 100644 --- a/docs/API.md +++ b/docs/API.md @@ -57,6 +57,7 @@ curl -H "Authorization: Bearer hlm_p_xxxxxxxxxxxxxxxxxxxxxxxx" https://hm.exampl | `GET /api/v1/alerts` | What is wrong now. Filter with `?severity=`, `?category=`, `?acknowledged=true\|false`, `?silenced=true\|false`. | | `GET /api/v1/incidents` | The record of alerts that began and ended. `?status=open\|resolved`, `?limit=` (1–500, default 100). | | `GET /api/v1/incidents/:id` | One incident with its timeline. | +| `GET /api/v1/incidents/:id/before?minutes=` | What changed before it began, ranked by how likely each is to be connected. `minutes` 15–1440 (120 by default). | | `GET /api/v1/inventory` | The hardware inventory. | | `GET /api/v1/network` | Network equipment read from UniFi: devices, networks, Wi-Fi names, and which server is plugged into which device. | | `GET /api/v1/endpoints` | The endpoint checks and how they are doing. | @@ -201,6 +202,33 @@ An *incident* is one time an alert was raised and later went away; the same aler `GET /api/v1/incidents/:id` adds the **timeline** (when it was raised, acknowledged, handed over, what was done to the thing it is about in the hour around it, when it resolved) and the notes people wrote. Open incidents are always listed; resolved ones newest first, up to `limit`. +### `GET /api/v1/incidents/:id/before` + +The "what changed before this?" list for one incident: what the app knows happened in the window before it began — settings and other changes in the audit log, maintenance windows that began or ended, update runs +and patch plans, other alerts raised first, and a step in the server's own figures — each scored and the best first. + +```json +{ "incidentId": 1, "openedAt": "2026-10-21T20:08:08.000Z", "windowMinutes": 120, + "from": "2026-10-21T18:08:08.000Z", "to": "2026-10-21T20:09:08.000Z", + "suspects": [ + { "at": "2026-10-21T20:02:08.000Z", "minutesBefore": 6, "kind": "maintenance", "title": "Maintenance on pve1 ended", "detail": "Kernel update", "actor": "Olga", + "relation": "needs", "about": "pve1 — which it needs", "why": "About something it needs directly, 6 min before", + "score": 86, "confidence": "likely", "link": "/maintenance" } + ], + "looked": { "auditEntries": 5, "maintenanceWindows": 1, "updateRuns": 1, "otherAlerts": 2, "series": 1 } } +``` + +| Field | Meaning | +|---|---| +| `kind` | `maintenance`, `update`, `alert` (another alert, or a host going down), `change`, `settings`, `metric` (a step in the server's own load, memory or disk) or `rule` (one of your alert rules was changed). | +| `relation` | How it is related to what the incident is about: `same`, `needs` (the host it runs on, the switch it is plugged into, something it depends on), `needed_by`, `everything` (a setting, DNS) or `other`. | +| `score`, `confidence` | 0–100, from how related it is, what kind of thing it is and how close in time. `likely` is 55 and over, `possible` 25 and over, otherwise `background`. Anything under 8 is left out. | +| `minutesBefore` | How long before the incident began. | +| `link` | A page in the app, as a path (not a full address). | +| `looked` | How much was looked through, so an empty `suspects` can say what it covered. An empty list means *nothing the app knows about*, not *nothing happened* — a change made on the machine itself is invisible to it. | + +At most ten suspects are returned. It is a list of suspects, not a diagnosis. + ### `GET /api/v1/inventory` ```json diff --git a/docs/MONITORING.md b/docs/MONITORING.md index 9d33268..f99ade7 100644 --- a/docs/MONITORING.md +++ b/docs/MONITORING.md @@ -244,6 +244,24 @@ timeline — what was done to the thing in the hour around it (maintenance, a re the page open; the same alert coming back is a **new** incident. Resolved incidents are kept **400 days**. They are on **Operations → Alerts → History**, in the read API (`/api/v1/incidents`) and in the monthly report. +### What changed before an incident + +Each incident page has a **What changed before this?** card (and `GET /api/v1/incidents/:id/before`): the things the app knows happened in the 30 minutes, 2, 6 or 24 hours before the alert began, +ranked. The sources are the audit log (settings, DNS, dependencies, integrations, a server being changed, a neighbour going offline or filling up), maintenance windows that began or ended, update runs and patch +plans, other alerts raised first, and a step in the server's own figures (a disk that jumped, memory that fell away). + +Each is scored from three things: + +| | How it counts | +|---|---| +| **How related** | The **same thing** the incident is about counts most; then **something it needs** (the host it runs on, the switch it is plugged into, a dependency you wrote — the nearest count more); then a **change that touches everything** (a setting, DNS); then **something that needs it**; and last something with **no link** the app knows of. | +| **What kind** | A maintenance window ending or a host going down is likelier than a change of settings, which is likelier than a note being written. Logins, notes, runbooks, tags and the like are not listed at all. A setting that bears on this kind of alert (thresholds, routing) counts for more. | +| **How close** | A change five minutes before counts for much more than one two hours before — the weight falls off over about an hour and never reaches zero inside the window. | + +They are shown as *Likely* (55 and over), *Possible* (25 and over) and *Background* (the rest, behind a button); under 8 is left out, and at most ten are shown. The audit entry "pve1 stopped reporting" +and the alert about it are one event and shown once. It is a **list of suspects, not a diagnosis**: it says what happened and when, not why, and a change made on the machine itself — a package installed by +hand, a config file edited — is not something the app can see. + ## History and how long it is kept | What | Kept | Grain | diff --git a/server/src/routes/incidents.ts b/server/src/routes/incidents.ts index 7651a3a..d5c6ebe 100644 --- a/server/src/routes/incidents.ts +++ b/server/src/routes/incidents.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { requireAuth, requireRole } from "../auth/middleware.js"; import { recordAudit } from "../services/audit.js"; import { addNote, getIncident, listIncidents, MAX_NOTE, removeNote } from "../services/incidents.js"; +import { whatChangedBefore } from "../services/incidentBefore.js"; import { asyncHandler } from "../utils/asyncHandler.js"; export const incidentsRouter = Router(); @@ -25,6 +26,15 @@ incidentsRouter.get("/:id", asyncHandler(async (req, res) => { res.json(detail); })); +// What changed in the minutes before it began, ranked by how likely each is to be connected. `?minutes=` 15–1440 (120 by default). +incidentsRouter.get("/:id/before", asyncHandler(async (req, res) => { + const id = idOf(req.params.id); + if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" }); + const report = await whatChangedBefore(id, req.query.minutes === undefined ? undefined : Number(req.query.minutes)); + if (!report) return res.status(404).json({ error: "not_found" }); + res.json(report); +})); + const noteSchema = z.object({ body: z.string().trim().min(1, "Write something first.").max(MAX_NOTE) }); incidentsRouter.post("/:id/notes", requireRole("operator"), asyncHandler(async (req, res) => { diff --git a/server/src/routes/publicApi.ts b/server/src/routes/publicApi.ts index b1729e8..c3940d8 100644 --- a/server/src/routes/publicApi.ts +++ b/server/src/routes/publicApi.ts @@ -5,6 +5,7 @@ import { endpointChecks, servers } from "../db/schema.js"; import { resolveApiToken, type ApiTokenRow, type ApiTokenUser } from "../services/apiTokens.js"; import { getEndpointHistory } from "../services/endpointChecks.js"; import { getIncident, listIncidents } from "../services/incidents.js"; +import { whatChangedBefore } from "../services/incidentBefore.js"; import { buildNetworkOverview } from "../services/networkOverview.js"; import { alertsV1, endpointsV1, inventoryV1, maintenanceV1, oneServerV1, serversV1 } from "../services/publicApi.js"; import { getServerMetrics, MAX_RANGE_DAYS } from "../services/serverMetrics.js"; @@ -67,6 +68,7 @@ publicApiRouter.get("/", (_req, res) => { "GET /api/v1/alerts?severity=&category=&acknowledged=&silenced=", "GET /api/v1/incidents?status=&limit=", "GET /api/v1/incidents/:id", + "GET /api/v1/incidents/:id/before?minutes=", "GET /api/v1/inventory", "GET /api/v1/network", "GET /api/v1/endpoints", @@ -135,6 +137,14 @@ publicApiRouter.get("/incidents/:id", asyncHandler(async (req, res) => { res.json(detail); })); +publicApiRouter.get("/incidents/:id/before", asyncHandler(async (req, res) => { + const id = idOf(req.params.id); + if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" }); + const report = await whatChangedBefore(id, req.query.minutes === undefined ? undefined : Number(req.query.minutes)); + if (!report) return res.status(404).json({ error: "not_found" }); + res.json(report); +})); + publicApiRouter.get("/inventory", asyncHandler(async (_req, res) => { res.json({ assets: await inventoryV1() }); })); diff --git a/server/src/services/incidentBefore.ts b/server/src/services/incidentBefore.ts new file mode 100644 index 0000000..a928aac --- /dev/null +++ b/server/src/services/incidentBefore.ts @@ -0,0 +1,416 @@ +/** + * "What changed before this?" — for one incident, the things the app knows happened in the minutes before it began, ranked by how likely each is to be connected. + * + * It looks at what is already recorded and nothing else: the audit log (settings, integrations, DNS, dependencies, a server being updated …), maintenance windows that + * began or ended, update runs and patch plans, other alerts that were raised first, and a sudden step in the figures of the server itself (a disk that jumped, memory that + * fell away). Each is scored from three things: + * + * how related it is — the same thing as the incident is about, something it needs (the host it runs on, the switch it is plugged into), something that needs it, a change + * that touches everything (a setting), or something with no link at all; + * what kind it is — a maintenance window ending or a host going down is a likelier cause than a runbook being edited; + * how close it was — a change five minutes before counts for much more than one two hours before. + * + * It is a list of *suspects*, not a diagnosis: the app can say what happened and when, not why. The pure part (scoring, ranking, describing the step in a series) is kept + * apart from the reading so it can be tested exactly. + */ +import { and, asc, desc, eq, gte, isNotNull, lte, ne, or } from "drizzle-orm"; +import { db } from "../db/client.js"; +import { alertIncidents, auditLog, maintenanceWindows, patchPlanRuns, serverMetrics, updateRuns } from "../db/schema.js"; +import { dependentsOf, loadGraph, needsOfNode, nodeKey, type NodeType } from "./dependencies.js"; +import { describeChange } from "./changeLog.js"; +import { auditTimeToIso } from "./serverActivity.js"; + +export const DEFAULT_MINUTES = 120; +export const MIN_MINUTES = 15; +export const MAX_MINUTES = 24 * 60; +export const MAX_SUSPECTS = 10; +/** Below this score a candidate is not worth showing. */ +export const MIN_SCORE = 8; + +export type SuspectKind = "maintenance" | "update" | "alert" | "change" | "settings" | "metric" | "rule"; +export type Relation = "same" | "needs" | "needed_by" | "everything" | "other"; +export type Confidence = "likely" | "possible" | "background"; + +export interface Suspect { + /** ISO. */ + at: string; + /** How long before the incident began, in minutes (0 for something at the same moment). */ + minutesBefore: number; + kind: SuspectKind; + title: string; + detail: string | null; + actor: string | null; + relation: Relation; + /** What it was about, in words: "this server", "pve1 — what it runs on". */ + about: string; + /** One line on why it is on the list. */ + why: string; + score: number; + confidence: Confidence; + /** A page in the app that shows more. */ + link: string | null; + /** For another alert that was raised first. */ + incidentId?: number; +} + +export interface Candidate { + at: number; + kind: SuspectKind; + title: string; + detail: string | null; + actor: string | null; + relation: Relation; + /** How many steps away, for a thing that is needed or needs it. */ + distance: number; + aboutName: string | null; + /** How suspicious this kind of thing is, 0–1. */ + weight: number; + link: string | null; + incidentId?: number; + /** Set on the audit entry for a server going offline or a disk filling, and on the alert about it: the same event, told twice (see `withoutRepeats`). */ + eventKey?: string; +} + +/** Pure: the audit entry "pve1 stopped reporting" and the alert about the same moment are one event; keep the alert, which can be opened. */ +export function withoutRepeats(candidates: Candidate[], withinMs = 5 * 60_000): Candidate[] { + const alerts = candidates.filter((c) => c.eventKey !== undefined && c.incidentId !== undefined); + return candidates.filter((c) => !(c.eventKey !== undefined && c.incidentId === undefined && alerts.some((a) => a.eventKey === c.eventKey && Math.abs(a.at - c.at) <= withinMs))); +} + +/** How related to the incident, 0–1. */ +export function relevance(relation: Relation, distance: number): number { + switch (relation) { + case "same": + return 1; + case "needs": + return distance <= 1 ? 0.95 : 0.75; + case "needed_by": + return distance <= 1 ? 0.5 : 0.35; + case "everything": + return 0.6; + default: + return 0.25; + } +} + +/** How much a change counts for, by how long before the incident it was: close is much, an hour is little, but never nothing inside the window. */ +export function recency(minutesBefore: number): number { + return Math.max(0.25, Math.exp(-Math.max(0, minutesBefore) / 60)); +} + +export const confidenceOf = (score: number): Confidence => (score >= 55 ? "likely" : score >= 25 ? "possible" : "background"); + +const RELATION_WORDS: Record string> = { + same: () => "this", + needs: (n) => `${n ?? "something it needs"} — which it needs`, + needed_by: (n) => `${n ?? "something that needs it"} — which needs it`, + everything: () => "the whole installation", + other: (n) => n ?? "something unrelated", +}; + +function whyOf(c: Candidate, minutesBefore: number): string { + const when = minutesBefore < 1 ? "at the same moment" : minutesBefore < 90 ? `${Math.round(minutesBefore)} min before` : `${Math.round((minutesBefore / 60) * 10) / 10} h before`; + const how = + c.relation === "same" + ? "About the same thing" + : c.relation === "needs" + ? c.distance <= 1 + ? "About something it needs directly" + : "About something further down the chain it needs" + : c.relation === "needed_by" + ? "About something that needs it" + : c.relation === "everything" + ? "A change that touches everything" + : "Not linked to it that the app knows"; + return `${how}, ${when}`; +} + +/** Pure: scores the candidates, drops the negligible, and puts the likeliest first (ties: the nearer one). */ +export function rankSuspects(candidates: Candidate[], openedAtMs: number, max = MAX_SUSPECTS): Suspect[] { + const out: Suspect[] = []; + for (const c of candidates) { + const minutesBefore = Math.max(0, (openedAtMs - c.at) / 60_000); + const score = Math.round(100 * relevance(c.relation, c.distance) * c.weight * recency(minutesBefore)); + if (score < MIN_SCORE) continue; + out.push({ + at: new Date(c.at).toISOString(), + minutesBefore: Math.round(minutesBefore * 10) / 10, + kind: c.kind, + title: c.title, + detail: c.detail, + actor: c.actor, + relation: c.relation, + about: RELATION_WORDS[c.relation](c.aboutName), + why: whyOf(c, minutesBefore), + score, + confidence: confidenceOf(score), + link: c.link, + ...(c.incidentId !== undefined ? { incidentId: c.incidentId } : {}), + }); + } + return out.sort((a, b) => b.score - a.score || a.minutesBefore - b.minutesBefore).slice(0, max); +} + +/** Pure: whether a series took a real step, and where. `points` oldest first. Needs three readings. */ +export function stepOf(points: { t: number; v: number }[], threshold: number): { from: number; to: number; at: number } | null { + if (points.length < 3) return null; + const first = points[0].v; + const last = points[points.length - 1].v; + if (Math.abs(last - first) < threshold) return null; + // Where the biggest single move was: that is when it happened. + let at = points[1].t; + let biggest = 0; + for (let i = 1; i < points.length; i++) { + const move = Math.abs(points[i].v - points[i - 1].v); + if (move > biggest) { + biggest = move; + at = points[i].t; + } + } + return { from: first, to: last, at }; +} + +// ─── What each kind of audit entry is worth ───────────────────────────────── + +interface AuditWeight { + weight: number; + kind: SuspectKind; +} + +/** + * Pure: how suspicious a recorded action is, or null when it is not worth listing (somebody logged in, a note was written, a runbook was edited). `sameThing` says + * the entry is about what the incident is about. + */ +export function auditWeight(category: string, action: string, ctx: { sameThing: boolean; neighbour: boolean; incidentCategory: string; changedSections: string[] }): AuditWeight | null { + switch (category) { + case "server": { + const health = action === "went_offline" || action === "disk_full"; + if (health) return ctx.sameThing ? null : { weight: 0.95, kind: "alert" }; // for this very thing it is only the incident again + if (action === "back_online" || action === "disk_ok") return ctx.sameThing ? null : { weight: 0.35, kind: "alert" }; + if (action === "agent_updated") return { weight: 0.7, kind: "change" }; + if (action === "update" || action === "rotate_token") return { weight: 0.5, kind: "change" }; + if (action === "create") return { weight: 0.3, kind: "change" }; + return null; + } + case "settings": { + const touches = ctx.changedSections.some((s) => ["healthChecks", "notifications", "routing", "escalation", "quietHours"].includes(s)); + const relevant = touches && ["offline", "disk", "services", "monitoring", "backup", "updates", "integration", "automation"].includes(ctx.incidentCategory); + return { weight: relevant ? 0.8 : 0.5, kind: "settings" }; + } + case "integration": + return ctx.sameThing || ctx.neighbour ? { weight: 0.8, kind: "change" } : null; + case "dns": + return { weight: ["monitoring", "offline", "expiry"].includes(ctx.incidentCategory) ? 0.45 : 0.3, kind: "change" }; + case "dependency": + return { weight: 0.5, kind: "change" }; + case "network": + case "ipam": + return { weight: 0.3, kind: "change" }; + case "secret": + case "domain": + return ctx.incidentCategory === "expiry" ? { weight: 0.4, kind: "change" } : null; + default: + return null; + } +} + +// ─── Reading ──────────────────────────────────────────────────────────────── + +export interface BeforeReport { + incidentId: number; + openedAt: string; + windowMinutes: number; + from: string; + to: string; + suspects: Suspect[]; + /** How much was looked through, so "nothing" can say what it looked at. */ + looked: { auditEntries: number; maintenanceWindows: number; updateRuns: number; otherAlerts: number; series: number }; +} + +const sqliteTime = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace("T", " "); + +function parse(detail: string | null): Record { + if (!detail) return {}; + try { + const v = JSON.parse(detail); + return v && typeof v === "object" ? (v as Record) : {}; + } catch { + return {}; + } +} + +export const clampMinutes = (v: unknown): number => { + const n = Math.round(Number(v)); + return Number.isFinite(n) ? Math.min(MAX_MINUTES, Math.max(MIN_MINUTES, n)) : DEFAULT_MINUTES; +}; + +/** The ranked list for one incident, or null when there is no such incident. */ +export async function whatChangedBefore(incidentId: number, minutes: number = DEFAULT_MINUTES, now: Date = new Date()): Promise { + const [inc] = await db.select().from(alertIncidents).where(eq(alertIncidents.id, incidentId)).limit(1); + if (!inc) return null; + const windowMinutes = clampMinutes(minutes); + const opened = Date.parse(inc.openedAt); + const fromMs = opened - windowMinutes * 60_000; + const toMs = Math.min(opened + 60_000, now.getTime()); + + // What is related to what: the thing the incident is about, what it needs and what needs it. + const graph = await loadGraph(); + const isNode = inc.targetType === "server" || inc.targetType === "integration" || inc.targetType === "device"; + const ownKey = isNode && inc.targetId !== null ? nodeKey(inc.targetType as NodeType, inc.targetId) : null; + const related = new Map(); + if (ownKey) { + related.set(ownKey, { relation: "same", distance: 0 }); + const direct = new Set(graph.edges.get(ownKey) ?? []); + for (const k of needsOfNode(graph.edges, ownKey)) related.set(k, { relation: "needs", distance: direct.has(k) ? 1 : 2 }); + for (const d of dependentsOf(graph.edges, ownKey)) if (!related.has(d.key)) related.set(d.key, { relation: "needed_by", distance: d.distance }); + } + const relationOf = (type: string | null, id: number | string | null): { relation: Relation; distance: number; name: string | null } => { + if (type === null || id === null) return { relation: "other", distance: 0, name: null }; + const key = `${type}:${id}`; + const r = related.get(key); + return r ? { ...r, name: graph.names.get(key) ?? null } : { relation: "other", distance: 0, name: graph.names.get(key) ?? null }; + }; + + const candidates: Candidate[] = []; + const looked = { auditEntries: 0, maintenanceWindows: 0, updateRuns: 0, otherAlerts: 0, series: 0 }; + + // The ids of the thing an alert is about when it is a check, a heartbeat or a rule of your own (so a change to *that* is the same thing). + const ownCheck = /^(endpoint|endpoint-cert|heartbeat|heartbeat-fail):(\d+)/.exec(inc.alertKey.replace(/^[a-z]+:/, "")); + const ownRule = /rule:(\d+):server:/.exec(inc.alertKey); + + // 1. The audit log. + const audits = await db.select().from(auditLog).where(and(gte(auditLog.createdAt, sqliteTime(fromMs)), lte(auditLog.createdAt, sqliteTime(toMs)))).orderBy(asc(auditLog.createdAt)).limit(1000); + looked.auditEntries = audits.length; + for (const a of audits) { + const d = parse(a.detail); + const at = Date.parse(auditTimeToIso(a.createdAt)); + if (a.category === "alert_rule") { + if (ownRule && a.targetId === ownRule[1]) { + const described = describeChange(a.category, a.action, d); + candidates.push({ at, kind: "rule", title: described.title, detail: described.detail, actor: a.actorLabel, relation: "same", distance: 0, aboutName: null, weight: 0.9, link: "/alert-rules" }); + } + continue; + } + if (a.category === "endpoint" || a.category === "heartbeat") { + if (ownCheck && a.targetId === ownCheck[2] && ((a.category === "endpoint" && ownCheck[1].startsWith("endpoint")) || (a.category === "heartbeat" && ownCheck[1].startsWith("heartbeat")))) { + const described = describeChange(a.category, a.action, d); + candidates.push({ at, kind: "change", title: described.title, detail: described.detail, actor: a.actorLabel, relation: "same", distance: 0, aboutName: null, weight: 0.9, link: a.category === "endpoint" ? "/endpoints" : "/heartbeats" }); + } + continue; + } + const rel = relationOf(a.targetType, a.targetId); + const global = ["settings", "dns", "dependency", "network", "ipam", "secret", "domain"].includes(a.category); + const changes = d.changes && typeof d.changes === "object" ? Object.keys(d.changes as Record) : Array.isArray(d.sections) ? (d.sections as unknown[]).filter((s): s is string => typeof s === "string") : []; + const w = auditWeight(a.category, a.action, { sameThing: rel.relation === "same", neighbour: rel.relation === "needs" || rel.relation === "needed_by", incidentCategory: inc.category, changedSections: changes }); + if (!w) continue; + // Something about another server that is not linked to this one is not a suspect, unless it is a health event of a neighbour. + if (!global && rel.relation === "other") continue; + const described = describeChange(a.category, a.action, d); + const relation: Relation = global && rel.relation === "other" ? "everything" : rel.relation; + candidates.push({ + at, + kind: w.kind, + title: described.title, + detail: described.detail, + actor: a.actorLabel, + relation, + distance: rel.distance, + aboutName: rel.name, + weight: w.weight, + eventKey: a.category === "server" && (a.action === "went_offline" || a.action === "disk_full") ? `${a.targetType}:${a.targetId}:${a.action === "went_offline" ? "offline" : "disk"}` : undefined, + link: a.targetType === "server" && a.targetId ? `/servers/${a.targetId}` : a.category === "settings" ? "/settings" : a.category === "integration" ? "/integrations" : a.category === "dns" ? "/dns" : a.category === "dependency" ? "/map?view=dependencies" : null, + }); + } + + // 2. Maintenance windows on the thing, what it needs or what needs it: ending is the likeliest (what was held back is let through), beginning next. + const windows = await db + .select() + .from(maintenanceWindows) + .where(or(and(gte(maintenanceWindows.endsAt, new Date(fromMs).toISOString()), lte(maintenanceWindows.endsAt, new Date(toMs).toISOString())), and(gte(maintenanceWindows.startedAt, new Date(fromMs).toISOString()), lte(maintenanceWindows.startedAt, new Date(toMs).toISOString())))); + for (const w of windows) { + if (w.endsAt <= w.startedAt) continue; // withdrawn before it began + const rel = relationOf(w.targetType, w.targetId); + if (rel.relation === "other") continue; + looked.maintenanceWindows++; + const name = rel.name ?? `${w.targetType} #${w.targetId}`; + const endMs = Date.parse(w.endsAt); + if (endMs >= fromMs && endMs <= toMs) { + candidates.push({ at: endMs, kind: "maintenance", title: `Maintenance on ${name} ended`, detail: w.reason, actor: w.createdBy, relation: rel.relation, distance: rel.distance, aboutName: rel.name, weight: 1, link: "/maintenance" }); + } + const startMs = Date.parse(w.startedAt); + if (startMs >= fromMs && startMs <= toMs) { + candidates.push({ at: startMs, kind: "maintenance", title: `Maintenance began on ${name} — alerts held back`, detail: w.reason, actor: w.createdBy, relation: rel.relation, distance: rel.distance, aboutName: rel.name, weight: 0.5, link: "/maintenance" }); + } + } + + // 3. Update runs and patch plans that touched it or something related. + const runs = await db + .select() + .from(updateRuns) + .where(or(and(gte(updateRuns.startedAt, new Date(fromMs).toISOString()), lte(updateRuns.startedAt, new Date(toMs).toISOString())), and(isNotNull(updateRuns.finishedAt), gte(updateRuns.finishedAt, new Date(fromMs).toISOString()), lte(updateRuns.finishedAt, new Date(toMs).toISOString())))); + for (const r of runs) { + const rel = relationOf("server", r.serverId); + if (rel.relation === "other") continue; + looked.updateRuns++; + const finishedMs = r.finishedAt ? Date.parse(r.finishedAt) : null; + const at = finishedMs !== null && finishedMs >= fromMs && finishedMs <= toMs ? finishedMs : Date.parse(r.startedAt); + const verb = finishedMs !== null && at === finishedMs ? (r.status === "done" ? "finished" : r.status === "failed" ? "failed" : "stopped") : "started"; + candidates.push({ at, kind: "update", title: `Update run ${verb} on ${rel.name ?? `server #${r.serverId}`}: ${r.templateName}`, detail: r.startedBy ? `started by ${r.startedBy}` : null, actor: r.startedBy, relation: rel.relation, distance: rel.distance, aboutName: rel.name, weight: 1, link: "/updates" }); + } + const plans = await db + .select() + .from(patchPlanRuns) + .where(or(and(gte(patchPlanRuns.startedAt, new Date(fromMs).toISOString()), lte(patchPlanRuns.startedAt, new Date(toMs).toISOString())), and(isNotNull(patchPlanRuns.finishedAt), gte(patchPlanRuns.finishedAt, new Date(fromMs).toISOString()), lte(patchPlanRuns.finishedAt, new Date(toMs).toISOString())))); + for (const p of plans) { + let items: { serverId?: number; name?: string }[] = []; + try { + const v = JSON.parse(p.items); + if (Array.isArray(v)) items = v; + } catch { + // an unreadable list simply matches nothing + } + const hits = items.map((i) => ({ i, rel: relationOf("server", i.serverId ?? null) })).filter((x) => x.rel.relation !== "other"); + if (hits.length === 0) continue; + looked.updateRuns++; + const best = hits.sort((x, y) => relevance(y.rel.relation, y.rel.distance) - relevance(x.rel.relation, x.rel.distance))[0]; + const finishedMs = p.finishedAt ? Date.parse(p.finishedAt) : null; + const at = finishedMs !== null && finishedMs >= fromMs && finishedMs <= toMs ? finishedMs : Date.parse(p.startedAt); + candidates.push({ at, kind: "update", title: `Patch plan "${p.planName}" ${finishedMs !== null && at === finishedMs ? (p.status === "done" ? "finished" : p.status) : "started"}`, detail: `${hits.length === 1 ? best.i.name ?? "A server" : `${hits.length} related servers`} in it`, actor: p.startedBy, relation: best.rel.relation, distance: best.rel.distance, aboutName: best.rel.name, weight: 1, link: "/updates" }); + } + + // 4. Other alerts that were raised first. + const others = await db + .select() + .from(alertIncidents) + .where(and(ne(alertIncidents.id, inc.id), gte(alertIncidents.openedAt, new Date(fromMs).toISOString()), lte(alertIncidents.openedAt, inc.openedAt))) + .orderBy(desc(alertIncidents.openedAt)) + .limit(200); + for (const o of others) { + looked.otherAlerts++; + const rel = relationOf(o.targetType, o.targetId); + // A dependant that went wrong first is not much of a cause; something it needs, or another problem on the same thing, is. + candidates.push({ at: Date.parse(o.openedAt), kind: "alert", title: `Another alert was raised: ${o.message}`, detail: `${o.category}, ${o.severity}`, actor: null, relation: rel.relation, distance: rel.distance, aboutName: rel.name ?? o.targetName, weight: rel.relation === "same" ? 0.55 : 0.9, link: `/incidents/${o.id}`, incidentId: o.id, eventKey: o.targetType && o.targetId !== null && (o.category === "offline" || o.category === "disk") ? `${o.targetType}:${o.targetId}:${o.category === "offline" ? "offline" : "disk"}` : undefined }); + } + + // 5. A step in the server's own figures. + if (inc.targetType === "server" && inc.targetId !== null) { + const rows = await db + .select() + .from(serverMetrics) + .where(and(eq(serverMetrics.serverId, inc.targetId), gte(serverMetrics.recordedAt, new Date(fromMs).toISOString()), lte(serverMetrics.recordedAt, new Date(toMs).toISOString()))) + .orderBy(asc(serverMetrics.recordedAt)); + const by = new Map(); + for (const r of rows) by.set(r.metric, [...(by.get(r.metric) ?? []), { t: Date.parse(r.recordedAt), v: r.value }]); + for (const [metric, points] of by) { + looked.series++; + const isDisk = metric.startsWith("disk:"); + const step = stepOf(points, isDisk ? 8 : 30); + if (!step) continue; + const what = metric === "cpu" ? "CPU load" : metric === "mem" ? "Memory in use" : `Disk ${metric.slice(5)}`; + const span = Math.round((points[points.length - 1].t - points[0].t) / 60_000); + candidates.push({ at: step.at, kind: "metric", title: `${what} ${step.to > step.from ? "rose" : "fell"} from ${Math.round(step.from)}% to ${Math.round(step.to)}%`, detail: `over the ${span} minutes before it began`, actor: null, relation: "same", distance: 0, aboutName: null, weight: isDisk ? 0.7 : 0.5, link: `/servers/${inc.targetId}?view=history` }); + } + } + + return { incidentId: inc.id, openedAt: inc.openedAt, windowMinutes, from: new Date(fromMs).toISOString(), to: new Date(toMs).toISOString(), suspects: rankSuspects(withoutRepeats(candidates), opened), looked }; +} diff --git a/server/test/incidentBefore.test.ts b/server/test/incidentBefore.test.ts new file mode 100644 index 0000000..1492b4e --- /dev/null +++ b/server/test/incidentBefore.test.ts @@ -0,0 +1,230 @@ +/** + * "What changed before this?": ranking what happened in the minutes before an incident, from the audit log, maintenance windows, update runs, other alerts and the + * server's own figures. + * + * Run with: npm test --workspace server + */ +import { test, describe, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AddressInfo } from "node:net"; +import type { Server } from "node:http"; + +const dir = mkdtempSync(join(tmpdir(), "hm-before-")); +process.env.DATABASE_PATH = join(dir, "test.sqlite"); +process.env.SESSION_DIR = join(dir, "sessions"); +process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); +process.env.HEALTH_STARTUP_GRACE_MS = "0"; + +const express = (await import("express")).default; +const { db } = await import("../src/db/client.js"); +const { users, servers, auditLog, maintenanceWindows, updateRuns, alertIncidents, serverMetrics, dependencies } = await import("../src/db/schema.js"); +const { runMigrations } = await import("../src/db/migrate.js"); +const b = await import("../src/services/incidentBefore.js"); +const { incidentsRouter } = await import("../src/routes/incidents.js"); + +await runMigrations(); + +const OPENED = Date.parse("2026-10-17T12:00:00.000Z"); +const ago = (minutes: number) => OPENED - minutes * 60_000; +const iso = (ms: number) => new Date(ms).toISOString(); +const sqlite = (ms: number) => iso(ms).slice(0, 19).replace("T", " "); + +const cand = (extra: Partial[0][number]> = {}): Parameters[0][number] => ({ at: ago(10), kind: "change", title: "x", detail: null, actor: null, relation: "same", distance: 0, aboutName: null, weight: 1, link: null, ...extra }); + +after(() => { + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // Windows keeps the database file locked until the process exits. + } +}); + +describe("how a suspect is scored", () => { + test("close and related beats far and unrelated; a change five minutes before counts for far more than one two hours before", () => { + assert.ok(b.recency(5) > 0.9 && b.recency(60) < 0.4 && b.recency(120) === 0.25 && b.recency(1000) === 0.25, "never nothing inside the window, never more than 1"); + assert.equal(b.recency(0), 1); + const r = (relation: Parameters[0], d = 0) => b.relevance(relation, d); + assert.ok(r("same") > r("needs", 1) && r("needs", 1) > r("needs", 2) && r("needs", 2) > r("everything") && r("everything") > r("needed_by", 1) && r("needed_by", 1) > r("needed_by", 2) && r("needed_by", 2) > r("other"), "the thing itself, what it needs, a change to everything, what needs it, nothing in common"); + assert.deepEqual([b.confidenceOf(90), b.confidenceOf(40), b.confidenceOf(10)], ["likely", "possible", "background"]); + }); + + test("they are ranked by score, the negligible are left out, and a tie goes to the nearer", () => { + const ranked = b.rankSuspects( + [ + cand({ title: "settings long ago", at: ago(115), relation: "everything", weight: 0.5 }), + cand({ title: "host maintenance ended", at: ago(5), relation: "needs", distance: 1, weight: 1 }), + cand({ title: "same, a bit earlier", at: ago(30), relation: "same", weight: 1 }), + cand({ title: "unrelated and old", at: ago(110), relation: "other", weight: 0.3 }), + ], + OPENED, + ); + assert.deepEqual(ranked.map((s) => s.title), ["host maintenance ended", "same, a bit earlier", "settings long ago"], "the unrelated old one scores under the floor"); + assert.equal(ranked[0].confidence, "likely"); + assert.match(ranked[0].why, /About something it needs directly, 5 min before/); + assert.equal(ranked[0].about, "something it needs — which it needs"); + assert.equal(ranked[2].confidence, "background"); + assert.equal(b.rankSuspects(Array.from({ length: 30 }, (_, i) => cand({ title: `c${i}`, at: ago(1 + i) })), OPENED).length, b.MAX_SUSPECTS); + }); + + test("a step in a series needs three readings and a real move, and is dated by the biggest single move", () => { + const series = (...v: number[]) => v.map((x, i) => ({ t: ago(60 - i * 10), v: x })); + assert.equal(b.stepOf(series(50, 71), 8), null, "two readings say nothing"); + assert.equal(b.stepOf(series(50, 52, 53), 8), null, "a small move"); + const step = b.stepOf(series(50, 51, 66, 71), 8)!; + assert.deepEqual([step.from, step.to, step.at], [50, 71, ago(40)]); + assert.equal(b.stepOf(series(80, 60, 40), 8)!.to, 40, "falling counts too"); + }); + + test("what an audit entry is worth: a login or a note is nothing; a host going down is a lot; a setting that bears on this alert is more than one that doesn't", () => { + const ctx = { sameThing: false, neighbour: false, incidentCategory: "offline", changedSections: [] as string[] }; + assert.equal(b.auditWeight("session", "login", ctx), null); + assert.equal(b.auditWeight("note", "save", ctx), null); + assert.equal(b.auditWeight("runbook", "create", ctx), null); + assert.equal(b.auditWeight("server", "went_offline", { ...ctx, sameThing: true }), null, "for the thing itself that is only the incident again"); + assert.equal(b.auditWeight("server", "went_offline", { ...ctx, neighbour: true })!.weight, 0.95); + assert.ok(b.auditWeight("settings", "update", { ...ctx, changedSections: ["healthChecks"] })!.weight > b.auditWeight("settings", "update", { ...ctx, changedSections: ["display"] })!.weight); + assert.equal(b.auditWeight("settings", "update", { ...ctx, incidentCategory: "expiry", changedSections: ["healthChecks"] })!.weight, 0.5, "not relevant to an expiry"); + assert.equal(b.auditWeight("integration", "update", ctx), null, "an unrelated integration"); + assert.equal(b.auditWeight("integration", "update", { ...ctx, neighbour: true })!.weight, 0.8); + }); + + test("a host going down is told once: the alert about it is kept and the audit entry for the same moment is dropped", () => { + const entry = cand({ title: "Stopped reporting", at: ago(3), eventKey: "server:4:offline" }); + const alert = cand({ title: "Another alert was raised", at: ago(4), eventKey: "server:4:offline", incidentId: 7 }); + assert.deepEqual(b.withoutRepeats([entry, alert]).map((c) => c.title), ["Another alert was raised"]); + assert.equal(b.withoutRepeats([entry]).length, 1, "with no alert to stand for it, the entry stays"); + assert.equal(b.withoutRepeats([entry, { ...alert, at: ago(30) }]).length, 2, "an alert half an hour apart is a different event"); + assert.equal(b.withoutRepeats([entry, { ...alert, eventKey: "server:5:offline" }]).length, 2, "and one about another server too"); + }); + + test("the window is kept to something sensible", () => { + assert.deepEqual([b.clampMinutes(undefined), b.clampMinutes("abc"), b.clampMinutes(1), b.clampMinutes(99999), b.clampMinutes(45)], [120, 120, 15, 1440, 45]); + }); +}); + +describe("reading what happened", () => { + let app: Server; + let base = ""; + let who = "viewer"; + let webId = 0; + let incidentId = 0; + let hostIncidentId = 0; + + before(async () => { + await db.insert(users).values([ + { oidcSub: "viewer", name: "Vic", role: "viewer" }, + { oidcSub: "operator", name: "Olga", role: "operator" }, + ]); + const mk = async (name: string, extra: Record = {}) => (await db.insert(servers).values({ name, apiTokenHash: `h-${name}`, apiTokenPrefix: name.slice(0, 3), ipAddresses: "[]", ...extra } as never).returning())[0]; + const host = await mk("pve1"); + const web = await mk("web", { parentServerId: host.id }); + const pi = await mk("pi"); + webId = web.id; + + // The incident: web went wrong, five minutes after its host did. + const inc = (await db.insert(alertIncidents).values({ alertKey: `disk:disk:server:${web.id}:/`, severity: "warning", peakSeverity: "warning", category: "offline", source: "Server", message: "web hasn't reported for 61 min", targetType: "server", targetId: web.id, targetName: "web", openedAt: iso(OPENED), lastSeenAt: iso(OPENED) }).returning())[0]; + incidentId = inc.id; + hostIncidentId = (await db.insert(alertIncidents).values({ alertKey: `offline:offline:server:${host.id}`, severity: "critical", peakSeverity: "critical", category: "offline", source: "Server", message: "pve1 hasn't reported for 70 min", targetType: "server", targetId: host.id, targetName: "pve1", openedAt: iso(ago(4)), lastSeenAt: iso(ago(4)) }).returning())[0].id; + await db.insert(alertIncidents).values({ alertKey: "disk:disk:server:pi:/", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "pi disk nearly full", targetType: "server", targetId: pi.id, targetName: "pi", openedAt: iso(ago(10)), lastSeenAt: iso(ago(10)) }); + + const audit = (ms: number, category: string, action: string, targetType: string | null, targetId: number | null, detail: unknown = {}) => + db.insert(auditLog).values({ actorLabel: "Olga", category, action, targetType, targetId: targetId === null ? null : String(targetId), detail: JSON.stringify(detail), createdAt: sqlite(ms) }); + await audit(ago(100), "settings", "update", "settings", null, { sections: ["healthChecks"], changes: { healthChecks: { serverOfflineMinutes: { from: 60, to: 30 } } } }); + await audit(ago(30), "dns", "create", "dns_provider", 1, { name: "wiki.example.lan" }); + await audit(ago(3), "server", "went_offline", "server", host.id, { message: "pve1 hasn't reported for 70 min" }); + await audit(ago(2), "server", "went_offline", "server", web.id, { message: "web hasn't reported" }); // this incident again + await audit(ago(8), "server", "update", "server", pi.id, {}); // an unrelated server + await audit(ago(7), "session", "login", "user", 1, {}); + await audit(ago(6), "note", "save", "server", web.id, {}); + + await db.insert(maintenanceWindows).values({ targetType: "server", targetId: host.id, startedAt: iso(ago(65)), endsAt: iso(ago(6)), reason: "Kernel update", createdBy: "Olga" }); + await db.insert(maintenanceWindows).values({ targetType: "server", targetId: pi.id, startedAt: iso(ago(30)), endsAt: iso(ago(5)), reason: "Unrelated", createdBy: "Olga" }); + await db.insert(updateRuns).values({ serverId: web.id, integrationId: 1, projectId: 1, templateId: 1, templateName: "apt upgrade", hostLimit: "web", status: "done", startedBy: "Olga", startedAt: iso(ago(40)), finishedAt: iso(ago(25)), windowIds: "[]" } as never); + + for (const [min, v] of [[100, 50], [70, 51], [45, 66], [20, 72], [5, 73]] as const) await db.insert(serverMetrics).values({ serverId: web.id, metric: "disk:/", value: v, recordedAt: iso(ago(min)) }); + for (const [min, v] of [[100, 12], [60, 14], [20, 13]] as const) await db.insert(serverMetrics).values({ serverId: web.id, metric: "cpu", value: v, recordedAt: iso(ago(min)) }); + + const a = express(); + a.use(express.json()); + a.use((req, _res, next) => { + (req as unknown as { session: unknown }).session = { user: { sub: who } }; + next(); + }); + a.use("/api/incidents", incidentsRouter); + app = await new Promise((resolve) => { + const s = a.listen(0, "127.0.0.1", () => resolve(s)); + }); + base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/incidents`; + }); + after(() => app.close()); + + test("the likeliest are what is closely related and close in time: the host going down, its maintenance ending, the update run on the server itself", async () => { + const report = (await b.whatChangedBefore(incidentId, 120, new Date(OPENED + 60_000)))!; + const titles = report.suspects.map((s) => s.title); + const top = report.suspects.slice(0, 4).map((s) => s.title); + assert.ok(top.some((t) => /Maintenance on pve1 ended/.test(t)), "the host's maintenance ended six minutes before"); + assert.ok(top.some((t) => /another alert was raised: pve1 hasn't reported/i.test(t)), "and its alert was raised first"); + assert.ok(top.some((t) => /Update run finished on web: apt upgrade/.test(t))); + assert.ok(titles.some((t) => /Disk \/ rose from 50% to 73%/.test(t)), "a step in the server's own disk"); + assert.ok(!titles.some((t) => /CPU load/.test(t)), "a flat figure is nothing"); + assert.ok(!titles.some((t) => /Stopped reporting/.test(t)), "the host going down is there once, as the alert, not twice"); + const host = report.suspects.find((s) => /Maintenance on pve1 ended/.test(s.title))!; + assert.deepEqual([host.relation, host.confidence, host.kind, host.minutesBefore], ["needs", "likely", "maintenance", 6]); + assert.match(host.why, /something it needs directly, 6 min before/); + assert.equal(report.suspects.find((s) => s.incidentId === hostIncidentId)!.kind, "alert"); + for (let i = 1; i < report.suspects.length; i++) assert.ok(report.suspects[i - 1].score >= report.suspects[i].score, "ranked"); + }); + + test("what has nothing to do with it is left out: the other server's update, the login, the note, this incident's own offline record, an unrelated server's maintenance", async () => { + const titles = (await b.whatChangedBefore(incidentId, 120, new Date(OPENED + 60_000)))!.suspects.map((s) => s.title).join("\n"); + assert.ok(!/Unrelated/.test(titles), "maintenance on another server"); + assert.ok(!/Details changed/.test(titles), "a change to the other server"); + assert.ok(!/hasn't reported for 61/.test(titles) && !/Stopped reporting/.test(titles.split("\n").filter((t) => /web/.test(t)).join("")), "not the incident itself"); + assert.ok(!/signed in|login/i.test(titles) && !/note/i.test(titles)); + }); + + test("a setting that was changed before is a suspect, and one that bears on this kind of alert counts for more; the further back the window, the more it finds", async () => { + const short = (await b.whatChangedBefore(incidentId, 30, new Date(OPENED + 60_000)))!; + assert.ok(!short.suspects.some((s) => s.kind === "settings"), "100 minutes back is outside 30"); + const wide = (await b.whatChangedBefore(incidentId, 240, new Date(OPENED + 60_000)))!; + const setting = wide.suspects.find((s) => s.kind === "settings"); + assert.ok(setting, "inside four hours"); + assert.equal(setting!.relation, "everything"); + assert.ok(setting!.confidence === "background" || setting!.confidence === "possible"); + assert.ok(wide.looked.auditEntries >= 6 && wide.looked.maintenanceWindows === 1 && wide.looked.updateRuns === 1 && wide.looked.otherAlerts === 2 && wide.looked.series === 2, JSON.stringify(wide.looked)); + }); + + test("an incident about something with nothing around it says so, with what it looked at", async () => { + const lonely = (await db.insert(alertIncidents).values({ alertKey: "expiry:secret:9", severity: "warning", peakSeverity: "warning", category: "expiry", source: "Secrets", message: "A secret expires", openedAt: iso(OPENED + 5 * 3600_000), lastSeenAt: iso(OPENED + 5 * 3600_000) }).returning())[0]; + const report = (await b.whatChangedBefore(lonely.id, 60, new Date(OPENED + 6 * 3600_000)))!; + assert.deepEqual(report.suspects, []); + assert.equal(report.looked.otherAlerts, 0); + }); + + test("a dependency makes a neighbour: what a server needs counts, what it does not need is nothing", async () => { + const [svc] = await db.select().from(servers).where((await import("drizzle-orm")).eq(servers.name, "pi")); + await db.insert(dependencies).values({ fromType: "server", fromId: webId, toType: "server", toId: svc.id }); + const report = (await b.whatChangedBefore(incidentId, 120, new Date(OPENED + 60_000)))!; + const pi = report.suspects.find((s) => /pi disk nearly full/.test(s.title))!; + assert.ok(pi, "pi is now something web needs"); + assert.equal(pi.relation, "needs"); + }); + + describe("the address", () => { + test("everyone signed in can ask; the window is read, an unknown incident is 404, and a bad id 400", async () => { + who = "viewer"; + const ok = await fetch(`${base}/${incidentId}/before?minutes=45`); + assert.equal(ok.status, 200); + const body = (await ok.json()) as any; + assert.deepEqual([body.incidentId, body.windowMinutes], [incidentId, 45]); + assert.ok(Array.isArray(body.suspects) && body.looked); + assert.equal(((await (await fetch(`${base}/${incidentId}/before?minutes=abc`)).json()) as any).windowMinutes, 120, "an unreadable window is the usual one"); + assert.equal(((await (await fetch(`${base}/${incidentId}/before?minutes=99999`)).json()) as any).windowMinutes, 1440); + assert.equal((await fetch(`${base}/999999/before`)).status, 404); + assert.equal((await fetch(`${base}/abc/before`)).status, 400); + }); + }); +}); diff --git a/server/test/readApi.test.ts b/server/test/readApi.test.ts index 66b896a..5b345cb 100644 --- a/server/test/readApi.test.ts +++ b/server/test/readApi.test.ts @@ -218,6 +218,9 @@ describe("what a token can read", () => { const incidents = (await (await api("/incidents?status=resolved", secret)).json()) as any; assert.equal(incidents.incidents.length, 1); assert.equal(((await (await api(`/incidents/${incidents.incidents[0].id}`, secret)).json()) as any).incident?.id ?? incidents.incidents[0].id, incidents.incidents[0].id); + const before = (await (await api(`/incidents/${incidents.incidents[0].id}/before?minutes=30`, secret)).json()) as any; + assert.deepEqual([before.incidentId, before.windowMinutes, Array.isArray(before.suspects)], [incidents.incidents[0].id, 30, true], "what changed before it, over the same token"); + assert.equal((await api("/incidents/999999/before", secret)).status, 404); const open = ((await (await api("/incidents?status=open", secret)).json()) as any).incidents; assert.deepEqual(open.map((i: any) => i.alertKey), [`offline:offline:server:${ids.db}`], "the quiet server's alert is an incident that is still open"); diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 860d62a..847808b 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -1614,6 +1614,32 @@ export interface IncidentSummary { notes: number; } +export interface BeforeSuspect { + at: string; + minutesBefore: number; + kind: "maintenance" | "update" | "alert" | "change" | "settings" | "metric" | "rule"; + title: string; + detail: string | null; + actor: string | null; + relation: "same" | "needs" | "needed_by" | "everything" | "other"; + about: string; + why: string; + score: number; + confidence: "likely" | "possible" | "background"; + link: string | null; + incidentId?: number; +} + +export interface BeforeReport { + incidentId: number; + openedAt: string; + windowMinutes: number; + from: string; + to: string; + suspects: BeforeSuspect[]; + looked: { auditEntries: number; maintenanceWindows: number; updateRuns: number; otherAlerts: number; series: number }; +} + export interface IncidentDetail { incident: IncidentSummary; ack: AlertAck | null; @@ -2648,6 +2674,7 @@ export const api = { incidents: { list: () => request<{ incidents: IncidentSummary[] }>("/api/incidents"), get: (id: number) => request(`/api/incidents/${id}`), + before: (id: number, minutes: number) => request(`/api/incidents/${id}/before?minutes=${minutes}`), addNote: (id: number, body: string) => request<{ id: number }>(`/api/incidents/${id}/notes`, { method: "POST", body: JSON.stringify({ body }) }), removeNote: (id: number, noteId: number) => request(`/api/incidents/${id}/notes/${noteId}`, { method: "DELETE" }), }, diff --git a/web/src/components/WhatChangedBefore.tsx b/web/src/components/WhatChangedBefore.tsx new file mode 100644 index 0000000..aeade2a --- /dev/null +++ b/web/src/components/WhatChangedBefore.tsx @@ -0,0 +1,135 @@ +import { useEffect, useState } from "react"; +import { Link } from "react-router-dom"; +import { api, type BeforeReport, type BeforeSuspect } from "../api/client"; +import { BlockSkeleton } from "./Skeleton"; +import { ErrorNotice } from "./Notice"; +import { formatDateTime } from "../utils/date"; +import { readableError } from "../utils/errors"; + +const RANGES = [ + { minutes: 30, label: "30 min" }, + { minutes: 120, label: "2 hours" }, + { minutes: 360, label: "6 hours" }, + { minutes: 1440, label: "24 hours" }, +]; + +const CONFIDENCE: Record = { + likely: { label: "Likely", className: "bg-red-lt text-red", title: "Close in time and closely related" }, + possible: { label: "Possible", className: "bg-yellow-lt text-yellow", title: "Either close in time or closely related" }, + background: { label: "Background", className: "bg-secondary-lt text-secondary", title: "Happened in the window, but little to link it" }, +}; + +const KIND: Record = { + maintenance: "Maintenance", + update: "Update", + alert: "Another alert", + change: "Change", + settings: "Setting", + metric: "Its own figures", + rule: "Alert rule", +}; + +const minutesWords = (m: number) => (m < 1 ? "at the same moment" : m < 90 ? `${Math.round(m)} min before` : `${Math.round((m / 60) * 10) / 10} h before`); + +/** + * "What changed before this?": what the app knows happened in the time before an incident began — a maintenance window ending, a host going down, an update run, a setting changed, + * a step in the server's own figures — ranked by how close it was and how related. A list of suspects, not a diagnosis: it says what happened and when, not why. + */ +export default function WhatChangedBefore({ incidentId, openedAt }: { incidentId: number; openedAt: string }) { + const [minutes, setMinutes] = useState(120); + const [report, setReport] = useState(null); + const [error, setError] = useState(null); + const [showBackground, setShowBackground] = useState(false); + + useEffect(() => { + let alive = true; + setReport(null); + api.incidents + .before(incidentId, minutes) + .then((r) => { + if (!alive) return; + setReport(r); + setError(null); + }) + .catch((err) => alive && setError(readableError(err))); + return () => { + alive = false; + }; + }, [incidentId, minutes]); + + const strong = report?.suspects.filter((s) => s.confidence !== "background") ?? []; + const background = report?.suspects.filter((s) => s.confidence === "background") ?? []; + const shown = [...strong, ...(showBackground || strong.length === 0 ? background : [])]; + + return ( +
+
+
+

What changed before this?

+
Before it began, {formatDateTime(new Date(openedAt))}
+
+
+
+ {RANGES.map((r) => ( + + ))} +
+
+
+ {error && ( +
+ +
+ )} + {!report && !error && ( +
+ +
+ )} + {report && shown.length === 0 && ( +
+ Nothing the app knows about changed in the {RANGES.find((r) => r.minutes === report.windowMinutes)?.label ?? `${report.windowMinutes} min`} before this. It looked through {report.looked.auditEntries} recorded{" "} + {report.looked.auditEntries === 1 ? "action" : "actions"}, {report.looked.maintenanceWindows} maintenance {report.looked.maintenanceWindows === 1 ? "window" : "windows"}, {report.looked.updateRuns} update{" "} + {report.looked.updateRuns === 1 ? "run" : "runs"}, {report.looked.otherAlerts} other {report.looked.otherAlerts === 1 ? "alert" : "alerts"} + {report.looked.series > 0 ? ` and ${report.looked.series} of the server's own figures` : ""} — a change made outside the app, such as one on the machine itself, is not something it can see. +
+ )} + {report && shown.length > 0 && ( +
+ {shown.map((s, i) => ( +
+
+ + {CONFIDENCE[s.confidence].label} + + {s.incidentId ? {s.title} : s.link ? {s.title} : s.title} +
+
+ {KIND[s.kind]} · {minutesWords(s.minutesBefore)} ({formatDateTime(new Date(s.at))}) + {s.actor && s.actor !== "system" ? ` · ${s.actor}` : ""} +
+ {s.detail &&
{s.detail}
} +
+ {s.why} + {s.relation !== "same" && s.relation !== "everything" ? ` · about ${s.about}` : s.relation === "everything" ? " · it touches everything" : ""} +
+
+ ))} +
+ )} + {report && background.length > 0 && strong.length > 0 && ( +
+ +
+ )} +
+ Ranked by how close each was and how related it is to this — the same thing, something it needs (the host it runs on), something that needs it, or a change that touches everything. These are suspects, + not a diagnosis: the app can say what happened and when, not why. +
+
+ ); +} diff --git a/web/src/pages/IncidentDetail.tsx b/web/src/pages/IncidentDetail.tsx index 96f85f0..fea355d 100644 --- a/web/src/pages/IncidentDetail.tsx +++ b/web/src/pages/IncidentDetail.tsx @@ -13,6 +13,7 @@ import { readableError } from "../utils/errors"; import { toast } from "../utils/toast"; import { CATEGORY_NAMES, durationWords, incidentPath, SEVERITY_BADGE } from "../components/IncidentHistory"; import { useBreadcrumbTail } from "../utils/breadcrumb"; +import WhatChangedBefore from "../components/WhatChangedBefore"; const KIND_ICON: Record = { alert: , @@ -193,6 +194,7 @@ export default function IncidentDetail({ user }: { user: CurrentUser }) {
+

What happened