What changed before this? on every incident, ranked

- A card on the incident page, and GET /api/incidents/:id/before and /api/v1/incidents/:id/before: what the app knows happened in the 30 min to 24 h before an alert began (audit log, maintenance windows, update runs and patch plans, other alerts, a step in the server's own figures), scored by how related, what kind and how close
- A host going down is shown once; Likely, Possible and Background; at most ten
- Docs and roles updated

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-21 20:25:45 +02:00
1 parent c2e76262ca
commit 0277126add
12 files changed
+887

No files matched your search

+10
View File
@@ -3,6 +3,7 @@ import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { addNote, getIncident, listIncidents, MAX_NOTE, removeNote } from "../services/incidents.js";
import { whatChangedBefore } from "../services/incidentBefore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const incidentsRouter = Router();
@@ -25,6 +26,15 @@ incidentsRouter.get("/:id", asyncHandler(async (req, res) => {
res.json(detail);
}));
// What changed in the minutes before it began, ranked by how likely each is to be connected. `?minutes=` 15–1440 (120 by default).
incidentsRouter.get("/:id/before", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const report = await whatChangedBefore(id, req.query.minutes === undefined ? undefined : Number(req.query.minutes));
if (!report) return res.status(404).json({ error: "not_found" });
res.json(report);
}));
const noteSchema = z.object({ body: z.string().trim().min(1, "Write something first.").max(MAX_NOTE) });
incidentsRouter.post("/:id/notes", requireRole("operator"), asyncHandler(async (req, res) => {
+10
View File
@@ -5,6 +5,7 @@ import { endpointChecks, servers } from "../db/schema.js";
import { resolveApiToken, type ApiTokenRow, type ApiTokenUser } from "../services/apiTokens.js";
import { getEndpointHistory } from "../services/endpointChecks.js";
import { getIncident, listIncidents } from "../services/incidents.js";
import { whatChangedBefore } from "../services/incidentBefore.js";
import { buildNetworkOverview } from "../services/networkOverview.js";
import { alertsV1, endpointsV1, inventoryV1, maintenanceV1, oneServerV1, serversV1 } from "../services/publicApi.js";
import { getServerMetrics, MAX_RANGE_DAYS } from "../services/serverMetrics.js";
@@ -67,6 +68,7 @@ publicApiRouter.get("/", (_req, res) => {
"GET /api/v1/alerts?severity=&category=&acknowledged=&silenced=",
"GET /api/v1/incidents?status=&limit=",
"GET /api/v1/incidents/:id",
"GET /api/v1/incidents/:id/before?minutes=",
"GET /api/v1/inventory",
"GET /api/v1/network",
"GET /api/v1/endpoints",
@@ -135,6 +137,14 @@ publicApiRouter.get("/incidents/:id", asyncHandler(async (req, res) => {
res.json(detail);
}));
publicApiRouter.get("/incidents/:id/before", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const report = await whatChangedBefore(id, req.query.minutes === undefined ? undefined : Number(req.query.minutes));
if (!report) return res.status(404).json({ error: "not_found" });
res.json(report);
}));
publicApiRouter.get("/inventory", asyncHandler(async (_req, res) => {
res.json({ assets: await inventoryV1() });
}));
+416
View File
@@ -0,0 +1,416 @@
/**
* "What changed before this?" — for one incident, the things the app knows happened in the minutes before it began, ranked by how likely each is to be connected.
*
* It looks at what is already recorded and nothing else: the audit log (settings, integrations, DNS, dependencies, a server being updated …), maintenance windows that
* began or ended, update runs and patch plans, other alerts that were raised first, and a sudden step in the figures of the server itself (a disk that jumped, memory that
* fell away). Each is scored from three things:
*
* how related it is — the same thing as the incident is about, something it needs (the host it runs on, the switch it is plugged into), something that needs it, a change
* that touches everything (a setting), or something with no link at all;
* what kind it is — a maintenance window ending or a host going down is a likelier cause than a runbook being edited;
* how close it was — a change five minutes before counts for much more than one two hours before.
*
* It is a list of *suspects*, not a diagnosis: the app can say what happened and when, not why. The pure part (scoring, ranking, describing the step in a series) is kept
* apart from the reading so it can be tested exactly.
*/
import { and, asc, desc, eq, gte, isNotNull, lte, ne, or } from "drizzle-orm";
import { db } from "../db/client.js";
import { alertIncidents, auditLog, maintenanceWindows, patchPlanRuns, serverMetrics, updateRuns } from "../db/schema.js";
import { dependentsOf, loadGraph, needsOfNode, nodeKey, type NodeType } from "./dependencies.js";
import { describeChange } from "./changeLog.js";
import { auditTimeToIso } from "./serverActivity.js";
export const DEFAULT_MINUTES = 120;
export const MIN_MINUTES = 15;
export const MAX_MINUTES = 24 * 60;
export const MAX_SUSPECTS = 10;
/** Below this score a candidate is not worth showing. */
export const MIN_SCORE = 8;
export type SuspectKind = "maintenance" | "update" | "alert" | "change" | "settings" | "metric" | "rule";
export type Relation = "same" | "needs" | "needed_by" | "everything" | "other";
export type Confidence = "likely" | "possible" | "background";
export interface Suspect {
/** ISO. */
at: string;
/** How long before the incident began, in minutes (0 for something at the same moment). */
minutesBefore: number;
kind: SuspectKind;
title: string;
detail: string | null;
actor: string | null;
relation: Relation;
/** What it was about, in words: "this server", "pve1 — what it runs on". */
about: string;
/** One line on why it is on the list. */
why: string;
score: number;
confidence: Confidence;
/** A page in the app that shows more. */
link: string | null;
/** For another alert that was raised first. */
incidentId?: number;
}
export interface Candidate {
at: number;
kind: SuspectKind;
title: string;
detail: string | null;
actor: string | null;
relation: Relation;
/** How many steps away, for a thing that is needed or needs it. */
distance: number;
aboutName: string | null;
/** How suspicious this kind of thing is, 0–1. */
weight: number;
link: string | null;
incidentId?: number;
/** Set on the audit entry for a server going offline or a disk filling, and on the alert about it: the same event, told twice (see `withoutRepeats`). */
eventKey?: string;
}
/** Pure: the audit entry "pve1 stopped reporting" and the alert about the same moment are one event; keep the alert, which can be opened. */
export function withoutRepeats(candidates: Candidate[], withinMs = 5 * 60_000): Candidate[] {
const alerts = candidates.filter((c) => c.eventKey !== undefined && c.incidentId !== undefined);
return candidates.filter((c) => !(c.eventKey !== undefined && c.incidentId === undefined && alerts.some((a) => a.eventKey === c.eventKey && Math.abs(a.at - c.at) <= withinMs)));
}
/** How related to the incident, 0–1. */
export function relevance(relation: Relation, distance: number): number {
switch (relation) {
case "same":
return 1;
case "needs":
return distance <= 1 ? 0.95 : 0.75;
case "needed_by":
return distance <= 1 ? 0.5 : 0.35;
case "everything":
return 0.6;
default:
return 0.25;
}
}
/** How much a change counts for, by how long before the incident it was: close is much, an hour is little, but never nothing inside the window. */
export function recency(minutesBefore: number): number {
return Math.max(0.25, Math.exp(-Math.max(0, minutesBefore) / 60));
}
export const confidenceOf = (score: number): Confidence => (score >= 55 ? "likely" : score >= 25 ? "possible" : "background");
const RELATION_WORDS: Record<Relation, (name: string | null) => string> = {
same: () => "this",
needs: (n) => `${n ?? "something it needs"} — which it needs`,
needed_by: (n) => `${n ?? "something that needs it"} — which needs it`,
everything: () => "the whole installation",
other: (n) => n ?? "something unrelated",
};
function whyOf(c: Candidate, minutesBefore: number): string {
const when = minutesBefore < 1 ? "at the same moment" : minutesBefore < 90 ? `${Math.round(minutesBefore)} min before` : `${Math.round((minutesBefore / 60) * 10) / 10} h before`;
const how =
c.relation === "same"
? "About the same thing"
: c.relation === "needs"
? c.distance <= 1
? "About something it needs directly"
: "About something further down the chain it needs"
: c.relation === "needed_by"
? "About something that needs it"
: c.relation === "everything"
? "A change that touches everything"
: "Not linked to it that the app knows";
return `${how}, ${when}`;
}
/** Pure: scores the candidates, drops the negligible, and puts the likeliest first (ties: the nearer one). */
export function rankSuspects(candidates: Candidate[], openedAtMs: number, max = MAX_SUSPECTS): Suspect[] {
const out: Suspect[] = [];
for (const c of candidates) {
const minutesBefore = Math.max(0, (openedAtMs - c.at) / 60_000);
const score = Math.round(100 * relevance(c.relation, c.distance) * c.weight * recency(minutesBefore));
if (score < MIN_SCORE) continue;
out.push({
at: new Date(c.at).toISOString(),
minutesBefore: Math.round(minutesBefore * 10) / 10,
kind: c.kind,
title: c.title,
detail: c.detail,
actor: c.actor,
relation: c.relation,
about: RELATION_WORDS[c.relation](c.aboutName),
why: whyOf(c, minutesBefore),
score,
confidence: confidenceOf(score),
link: c.link,
...(c.incidentId !== undefined ? { incidentId: c.incidentId } : {}),
});
}
return out.sort((a, b) => b.score - a.score || a.minutesBefore - b.minutesBefore).slice(0, max);
}
/** Pure: whether a series took a real step, and where. `points` oldest first. Needs three readings. */
export function stepOf(points: { t: number; v: number }[], threshold: number): { from: number; to: number; at: number } | null {
if (points.length < 3) return null;
const first = points[0].v;
const last = points[points.length - 1].v;
if (Math.abs(last - first) < threshold) return null;
// Where the biggest single move was: that is when it happened.
let at = points[1].t;
let biggest = 0;
for (let i = 1; i < points.length; i++) {
const move = Math.abs(points[i].v - points[i - 1].v);
if (move > biggest) {
biggest = move;
at = points[i].t;
}
}
return { from: first, to: last, at };
}
// ─── What each kind of audit entry is worth ─────────────────────────────────
interface AuditWeight {
weight: number;
kind: SuspectKind;
}
/**
* Pure: how suspicious a recorded action is, or null when it is not worth listing (somebody logged in, a note was written, a runbook was edited). `sameThing` says
* the entry is about what the incident is about.
*/
export function auditWeight(category: string, action: string, ctx: { sameThing: boolean; neighbour: boolean; incidentCategory: string; changedSections: string[] }): AuditWeight | null {
switch (category) {
case "server": {
const health = action === "went_offline" || action === "disk_full";
if (health) return ctx.sameThing ? null : { weight: 0.95, kind: "alert" }; // for this very thing it is only the incident again
if (action === "back_online" || action === "disk_ok") return ctx.sameThing ? null : { weight: 0.35, kind: "alert" };
if (action === "agent_updated") return { weight: 0.7, kind: "change" };
if (action === "update" || action === "rotate_token") return { weight: 0.5, kind: "change" };
if (action === "create") return { weight: 0.3, kind: "change" };
return null;
}
case "settings": {
const touches = ctx.changedSections.some((s) => ["healthChecks", "notifications", "routing", "escalation", "quietHours"].includes(s));
const relevant = touches && ["offline", "disk", "services", "monitoring", "backup", "updates", "integration", "automation"].includes(ctx.incidentCategory);
return { weight: relevant ? 0.8 : 0.5, kind: "settings" };
}
case "integration":
return ctx.sameThing || ctx.neighbour ? { weight: 0.8, kind: "change" } : null;
case "dns":
return { weight: ["monitoring", "offline", "expiry"].includes(ctx.incidentCategory) ? 0.45 : 0.3, kind: "change" };
case "dependency":
return { weight: 0.5, kind: "change" };
case "network":
case "ipam":
return { weight: 0.3, kind: "change" };
case "secret":
case "domain":
return ctx.incidentCategory === "expiry" ? { weight: 0.4, kind: "change" } : null;
default:
return null;
}
}
// ─── Reading ────────────────────────────────────────────────────────────────
export interface BeforeReport {
incidentId: number;
openedAt: string;
windowMinutes: number;
from: string;
to: string;
suspects: Suspect[];
/** How much was looked through, so "nothing" can say what it looked at. */
looked: { auditEntries: number; maintenanceWindows: number; updateRuns: number; otherAlerts: number; series: number };
}
const sqliteTime = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace("T", " ");
function parse(detail: string | null): Record<string, unknown> {
if (!detail) return {};
try {
const v = JSON.parse(detail);
return v && typeof v === "object" ? (v as Record<string, unknown>) : {};
} catch {
return {};
}
}
export const clampMinutes = (v: unknown): number => {
const n = Math.round(Number(v));
return Number.isFinite(n) ? Math.min(MAX_MINUTES, Math.max(MIN_MINUTES, n)) : DEFAULT_MINUTES;
};
/** The ranked list for one incident, or null when there is no such incident. */
export async function whatChangedBefore(incidentId: number, minutes: number = DEFAULT_MINUTES, now: Date = new Date()): Promise<BeforeReport | null> {
const [inc] = await db.select().from(alertIncidents).where(eq(alertIncidents.id, incidentId)).limit(1);
if (!inc) return null;
const windowMinutes = clampMinutes(minutes);
const opened = Date.parse(inc.openedAt);
const fromMs = opened - windowMinutes * 60_000;
const toMs = Math.min(opened + 60_000, now.getTime());
// What is related to what: the thing the incident is about, what it needs and what needs it.
const graph = await loadGraph();
const isNode = inc.targetType === "server" || inc.targetType === "integration" || inc.targetType === "device";
const ownKey = isNode && inc.targetId !== null ? nodeKey(inc.targetType as NodeType, inc.targetId) : null;
const related = new Map<string, { relation: Relation; distance: number }>();
if (ownKey) {
related.set(ownKey, { relation: "same", distance: 0 });
const direct = new Set(graph.edges.get(ownKey) ?? []);
for (const k of needsOfNode(graph.edges, ownKey)) related.set(k, { relation: "needs", distance: direct.has(k) ? 1 : 2 });
for (const d of dependentsOf(graph.edges, ownKey)) if (!related.has(d.key)) related.set(d.key, { relation: "needed_by", distance: d.distance });
}
const relationOf = (type: string | null, id: number | string | null): { relation: Relation; distance: number; name: string | null } => {
if (type === null || id === null) return { relation: "other", distance: 0, name: null };
const key = `${type}:${id}`;
const r = related.get(key);
return r ? { ...r, name: graph.names.get(key) ?? null } : { relation: "other", distance: 0, name: graph.names.get(key) ?? null };
};
const candidates: Candidate[] = [];
const looked = { auditEntries: 0, maintenanceWindows: 0, updateRuns: 0, otherAlerts: 0, series: 0 };
// The ids of the thing an alert is about when it is a check, a heartbeat or a rule of your own (so a change to *that* is the same thing).
const ownCheck = /^(endpoint|endpoint-cert|heartbeat|heartbeat-fail):(\d+)/.exec(inc.alertKey.replace(/^[a-z]+:/, ""));
const ownRule = /rule:(\d+):server:/.exec(inc.alertKey);
// 1. The audit log.
const audits = await db.select().from(auditLog).where(and(gte(auditLog.createdAt, sqliteTime(fromMs)), lte(auditLog.createdAt, sqliteTime(toMs)))).orderBy(asc(auditLog.createdAt)).limit(1000);
looked.auditEntries = audits.length;
for (const a of audits) {
const d = parse(a.detail);
const at = Date.parse(auditTimeToIso(a.createdAt));
if (a.category === "alert_rule") {
if (ownRule && a.targetId === ownRule[1]) {
const described = describeChange(a.category, a.action, d);
candidates.push({ at, kind: "rule", title: described.title, detail: described.detail, actor: a.actorLabel, relation: "same", distance: 0, aboutName: null, weight: 0.9, link: "/alert-rules" });
}
continue;
}
if (a.category === "endpoint" || a.category === "heartbeat") {
if (ownCheck && a.targetId === ownCheck[2] && ((a.category === "endpoint" && ownCheck[1].startsWith("endpoint")) || (a.category === "heartbeat" && ownCheck[1].startsWith("heartbeat")))) {
const described = describeChange(a.category, a.action, d);
candidates.push({ at, kind: "change", title: described.title, detail: described.detail, actor: a.actorLabel, relation: "same", distance: 0, aboutName: null, weight: 0.9, link: a.category === "endpoint" ? "/endpoints" : "/heartbeats" });
}
continue;
}
const rel = relationOf(a.targetType, a.targetId);
const global = ["settings", "dns", "dependency", "network", "ipam", "secret", "domain"].includes(a.category);
const changes = d.changes && typeof d.changes === "object" ? Object.keys(d.changes as Record<string, unknown>) : Array.isArray(d.sections) ? (d.sections as unknown[]).filter((s): s is string => typeof s === "string") : [];
const w = auditWeight(a.category, a.action, { sameThing: rel.relation === "same", neighbour: rel.relation === "needs" || rel.relation === "needed_by", incidentCategory: inc.category, changedSections: changes });
if (!w) continue;
// Something about another server that is not linked to this one is not a suspect, unless it is a health event of a neighbour.
if (!global && rel.relation === "other") continue;
const described = describeChange(a.category, a.action, d);
const relation: Relation = global && rel.relation === "other" ? "everything" : rel.relation;
candidates.push({
at,
kind: w.kind,
title: described.title,
detail: described.detail,
actor: a.actorLabel,
relation,
distance: rel.distance,
aboutName: rel.name,
weight: w.weight,
eventKey: a.category === "server" && (a.action === "went_offline" || a.action === "disk_full") ? `${a.targetType}:${a.targetId}:${a.action === "went_offline" ? "offline" : "disk"}` : undefined,
link: a.targetType === "server" && a.targetId ? `/servers/${a.targetId}` : a.category === "settings" ? "/settings" : a.category === "integration" ? "/integrations" : a.category === "dns" ? "/dns" : a.category === "dependency" ? "/map?view=dependencies" : null,
});
}
// 2. Maintenance windows on the thing, what it needs or what needs it: ending is the likeliest (what was held back is let through), beginning next.
const windows = await db
.select()
.from(maintenanceWindows)
.where(or(and(gte(maintenanceWindows.endsAt, new Date(fromMs).toISOString()), lte(maintenanceWindows.endsAt, new Date(toMs).toISOString())), and(gte(maintenanceWindows.startedAt, new Date(fromMs).toISOString()), lte(maintenanceWindows.startedAt, new Date(toMs).toISOString()))));
for (const w of windows) {
if (w.endsAt <= w.startedAt) continue; // withdrawn before it began
const rel = relationOf(w.targetType, w.targetId);
if (rel.relation === "other") continue;
looked.maintenanceWindows++;
const name = rel.name ?? `${w.targetType} #${w.targetId}`;
const endMs = Date.parse(w.endsAt);
if (endMs >= fromMs && endMs <= toMs) {
candidates.push({ at: endMs, kind: "maintenance", title: `Maintenance on ${name} ended`, detail: w.reason, actor: w.createdBy, relation: rel.relation, distance: rel.distance, aboutName: rel.name, weight: 1, link: "/maintenance" });
}
const startMs = Date.parse(w.startedAt);
if (startMs >= fromMs && startMs <= toMs) {
candidates.push({ at: startMs, kind: "maintenance", title: `Maintenance began on ${name} — alerts held back`, detail: w.reason, actor: w.createdBy, relation: rel.relation, distance: rel.distance, aboutName: rel.name, weight: 0.5, link: "/maintenance" });
}
}
// 3. Update runs and patch plans that touched it or something related.
const runs = await db
.select()
.from(updateRuns)
.where(or(and(gte(updateRuns.startedAt, new Date(fromMs).toISOString()), lte(updateRuns.startedAt, new Date(toMs).toISOString())), and(isNotNull(updateRuns.finishedAt), gte(updateRuns.finishedAt, new Date(fromMs).toISOString()), lte(updateRuns.finishedAt, new Date(toMs).toISOString()))));
for (const r of runs) {
const rel = relationOf("server", r.serverId);
if (rel.relation === "other") continue;
looked.updateRuns++;
const finishedMs = r.finishedAt ? Date.parse(r.finishedAt) : null;
const at = finishedMs !== null && finishedMs >= fromMs && finishedMs <= toMs ? finishedMs : Date.parse(r.startedAt);
const verb = finishedMs !== null && at === finishedMs ? (r.status === "done" ? "finished" : r.status === "failed" ? "failed" : "stopped") : "started";
candidates.push({ at, kind: "update", title: `Update run ${verb} on ${rel.name ?? `server #${r.serverId}`}: ${r.templateName}`, detail: r.startedBy ? `started by ${r.startedBy}` : null, actor: r.startedBy, relation: rel.relation, distance: rel.distance, aboutName: rel.name, weight: 1, link: "/updates" });
}
const plans = await db
.select()
.from(patchPlanRuns)
.where(or(and(gte(patchPlanRuns.startedAt, new Date(fromMs).toISOString()), lte(patchPlanRuns.startedAt, new Date(toMs).toISOString())), and(isNotNull(patchPlanRuns.finishedAt), gte(patchPlanRuns.finishedAt, new Date(fromMs).toISOString()), lte(patchPlanRuns.finishedAt, new Date(toMs).toISOString()))));
for (const p of plans) {
let items: { serverId?: number; name?: string }[] = [];
try {
const v = JSON.parse(p.items);
if (Array.isArray(v)) items = v;
} catch {
// an unreadable list simply matches nothing
}
const hits = items.map((i) => ({ i, rel: relationOf("server", i.serverId ?? null) })).filter((x) => x.rel.relation !== "other");
if (hits.length === 0) continue;
looked.updateRuns++;
const best = hits.sort((x, y) => relevance(y.rel.relation, y.rel.distance) - relevance(x.rel.relation, x.rel.distance))[0];
const finishedMs = p.finishedAt ? Date.parse(p.finishedAt) : null;
const at = finishedMs !== null && finishedMs >= fromMs && finishedMs <= toMs ? finishedMs : Date.parse(p.startedAt);
candidates.push({ at, kind: "update", title: `Patch plan "${p.planName}" ${finishedMs !== null && at === finishedMs ? (p.status === "done" ? "finished" : p.status) : "started"}`, detail: `${hits.length === 1 ? best.i.name ?? "A server" : `${hits.length} related servers`} in it`, actor: p.startedBy, relation: best.rel.relation, distance: best.rel.distance, aboutName: best.rel.name, weight: 1, link: "/updates" });
}
// 4. Other alerts that were raised first.
const others = await db
.select()
.from(alertIncidents)
.where(and(ne(alertIncidents.id, inc.id), gte(alertIncidents.openedAt, new Date(fromMs).toISOString()), lte(alertIncidents.openedAt, inc.openedAt)))
.orderBy(desc(alertIncidents.openedAt))
.limit(200);
for (const o of others) {
looked.otherAlerts++;
const rel = relationOf(o.targetType, o.targetId);
// A dependant that went wrong first is not much of a cause; something it needs, or another problem on the same thing, is.
candidates.push({ at: Date.parse(o.openedAt), kind: "alert", title: `Another alert was raised: ${o.message}`, detail: `${o.category}, ${o.severity}`, actor: null, relation: rel.relation, distance: rel.distance, aboutName: rel.name ?? o.targetName, weight: rel.relation === "same" ? 0.55 : 0.9, link: `/incidents/${o.id}`, incidentId: o.id, eventKey: o.targetType && o.targetId !== null && (o.category === "offline" || o.category === "disk") ? `${o.targetType}:${o.targetId}:${o.category === "offline" ? "offline" : "disk"}` : undefined });
}
// 5. A step in the server's own figures.
if (inc.targetType === "server" && inc.targetId !== null) {
const rows = await db
.select()
.from(serverMetrics)
.where(and(eq(serverMetrics.serverId, inc.targetId), gte(serverMetrics.recordedAt, new Date(fromMs).toISOString()), lte(serverMetrics.recordedAt, new Date(toMs).toISOString())))
.orderBy(asc(serverMetrics.recordedAt));
const by = new Map<string, { t: number; v: number }[]>();
for (const r of rows) by.set(r.metric, [...(by.get(r.metric) ?? []), { t: Date.parse(r.recordedAt), v: r.value }]);
for (const [metric, points] of by) {
looked.series++;
const isDisk = metric.startsWith("disk:");
const step = stepOf(points, isDisk ? 8 : 30);
if (!step) continue;
const what = metric === "cpu" ? "CPU load" : metric === "mem" ? "Memory in use" : `Disk ${metric.slice(5)}`;
const span = Math.round((points[points.length - 1].t - points[0].t) / 60_000);
candidates.push({ at: step.at, kind: "metric", title: `${what} ${step.to > step.from ? "rose" : "fell"} from ${Math.round(step.from)}% to ${Math.round(step.to)}%`, detail: `over the ${span} minutes before it began`, actor: null, relation: "same", distance: 0, aboutName: null, weight: isDisk ? 0.7 : 0.5, link: `/servers/${inc.targetId}?view=history` });
}
}
return { incidentId: inc.id, openedAt: inc.openedAt, windowMinutes, from: new Date(fromMs).toISOString(), to: new Date(toMs).toISOString(), suspects: rankSuspects(withoutRepeats(candidates), opened), looked };
}
+230
View File
@@ -0,0 +1,230 @@
/**
* "What changed before this?": ranking what happened in the minutes before an incident, from the audit log, maintenance windows, update runs, other alerts and the
* server's own figures.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-before-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, auditLog, maintenanceWindows, updateRuns, alertIncidents, serverMetrics, dependencies } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const b = await import("../src/services/incidentBefore.js");
const { incidentsRouter } = await import("../src/routes/incidents.js");
await runMigrations();
const OPENED = Date.parse("2026-10-17T12:00:00.000Z");
const ago = (minutes: number) => OPENED - minutes * 60_000;
const iso = (ms: number) => new Date(ms).toISOString();
const sqlite = (ms: number) => iso(ms).slice(0, 19).replace("T", " ");
const cand = (extra: Partial<Parameters<typeof b.rankSuspects>[0][number]> = {}): Parameters<typeof b.rankSuspects>[0][number] => ({ at: ago(10), kind: "change", title: "x", detail: null, actor: null, relation: "same", distance: 0, aboutName: null, weight: 1, link: null, ...extra });
after(() => {
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
describe("how a suspect is scored", () => {
test("close and related beats far and unrelated; a change five minutes before counts for far more than one two hours before", () => {
assert.ok(b.recency(5) > 0.9 && b.recency(60) < 0.4 && b.recency(120) === 0.25 && b.recency(1000) === 0.25, "never nothing inside the window, never more than 1");
assert.equal(b.recency(0), 1);
const r = (relation: Parameters<typeof b.relevance>[0], d = 0) => b.relevance(relation, d);
assert.ok(r("same") > r("needs", 1) && r("needs", 1) > r("needs", 2) && r("needs", 2) > r("everything") && r("everything") > r("needed_by", 1) && r("needed_by", 1) > r("needed_by", 2) && r("needed_by", 2) > r("other"), "the thing itself, what it needs, a change to everything, what needs it, nothing in common");
assert.deepEqual([b.confidenceOf(90), b.confidenceOf(40), b.confidenceOf(10)], ["likely", "possible", "background"]);
});
test("they are ranked by score, the negligible are left out, and a tie goes to the nearer", () => {
const ranked = b.rankSuspects(
[
cand({ title: "settings long ago", at: ago(115), relation: "everything", weight: 0.5 }),
cand({ title: "host maintenance ended", at: ago(5), relation: "needs", distance: 1, weight: 1 }),
cand({ title: "same, a bit earlier", at: ago(30), relation: "same", weight: 1 }),
cand({ title: "unrelated and old", at: ago(110), relation: "other", weight: 0.3 }),
],
OPENED,
);
assert.deepEqual(ranked.map((s) => s.title), ["host maintenance ended", "same, a bit earlier", "settings long ago"], "the unrelated old one scores under the floor");
assert.equal(ranked[0].confidence, "likely");
assert.match(ranked[0].why, /About something it needs directly, 5 min before/);
assert.equal(ranked[0].about, "something it needs — which it needs");
assert.equal(ranked[2].confidence, "background");
assert.equal(b.rankSuspects(Array.from({ length: 30 }, (_, i) => cand({ title: `c${i}`, at: ago(1 + i) })), OPENED).length, b.MAX_SUSPECTS);
});
test("a step in a series needs three readings and a real move, and is dated by the biggest single move", () => {
const series = (...v: number[]) => v.map((x, i) => ({ t: ago(60 - i * 10), v: x }));
assert.equal(b.stepOf(series(50, 71), 8), null, "two readings say nothing");
assert.equal(b.stepOf(series(50, 52, 53), 8), null, "a small move");
const step = b.stepOf(series(50, 51, 66, 71), 8)!;
assert.deepEqual([step.from, step.to, step.at], [50, 71, ago(40)]);
assert.equal(b.stepOf(series(80, 60, 40), 8)!.to, 40, "falling counts too");
});
test("what an audit entry is worth: a login or a note is nothing; a host going down is a lot; a setting that bears on this alert is more than one that doesn't", () => {
const ctx = { sameThing: false, neighbour: false, incidentCategory: "offline", changedSections: [] as string[] };
assert.equal(b.auditWeight("session", "login", ctx), null);
assert.equal(b.auditWeight("note", "save", ctx), null);
assert.equal(b.auditWeight("runbook", "create", ctx), null);
assert.equal(b.auditWeight("server", "went_offline", { ...ctx, sameThing: true }), null, "for the thing itself that is only the incident again");
assert.equal(b.auditWeight("server", "went_offline", { ...ctx, neighbour: true })!.weight, 0.95);
assert.ok(b.auditWeight("settings", "update", { ...ctx, changedSections: ["healthChecks"] })!.weight > b.auditWeight("settings", "update", { ...ctx, changedSections: ["display"] })!.weight);
assert.equal(b.auditWeight("settings", "update", { ...ctx, incidentCategory: "expiry", changedSections: ["healthChecks"] })!.weight, 0.5, "not relevant to an expiry");
assert.equal(b.auditWeight("integration", "update", ctx), null, "an unrelated integration");
assert.equal(b.auditWeight("integration", "update", { ...ctx, neighbour: true })!.weight, 0.8);
});
test("a host going down is told once: the alert about it is kept and the audit entry for the same moment is dropped", () => {
const entry = cand({ title: "Stopped reporting", at: ago(3), eventKey: "server:4:offline" });
const alert = cand({ title: "Another alert was raised", at: ago(4), eventKey: "server:4:offline", incidentId: 7 });
assert.deepEqual(b.withoutRepeats([entry, alert]).map((c) => c.title), ["Another alert was raised"]);
assert.equal(b.withoutRepeats([entry]).length, 1, "with no alert to stand for it, the entry stays");
assert.equal(b.withoutRepeats([entry, { ...alert, at: ago(30) }]).length, 2, "an alert half an hour apart is a different event");
assert.equal(b.withoutRepeats([entry, { ...alert, eventKey: "server:5:offline" }]).length, 2, "and one about another server too");
});
test("the window is kept to something sensible", () => {
assert.deepEqual([b.clampMinutes(undefined), b.clampMinutes("abc"), b.clampMinutes(1), b.clampMinutes(99999), b.clampMinutes(45)], [120, 120, 15, 1440, 45]);
});
});
describe("reading what happened", () => {
let app: Server;
let base = "";
let who = "viewer";
let webId = 0;
let incidentId = 0;
let hostIncidentId = 0;
before(async () => {
await db.insert(users).values([
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
{ oidcSub: "operator", name: "Olga", role: "operator" },
]);
const mk = async (name: string, extra: Record<string, unknown> = {}) => (await db.insert(servers).values({ name, apiTokenHash: `h-${name}`, apiTokenPrefix: name.slice(0, 3), ipAddresses: "[]", ...extra } as never).returning())[0];
const host = await mk("pve1");
const web = await mk("web", { parentServerId: host.id });
const pi = await mk("pi");
webId = web.id;
// The incident: web went wrong, five minutes after its host did.
const inc = (await db.insert(alertIncidents).values({ alertKey: `disk:disk:server:${web.id}:/`, severity: "warning", peakSeverity: "warning", category: "offline", source: "Server", message: "web hasn't reported for 61 min", targetType: "server", targetId: web.id, targetName: "web", openedAt: iso(OPENED), lastSeenAt: iso(OPENED) }).returning())[0];
incidentId = inc.id;
hostIncidentId = (await db.insert(alertIncidents).values({ alertKey: `offline:offline:server:${host.id}`, severity: "critical", peakSeverity: "critical", category: "offline", source: "Server", message: "pve1 hasn't reported for 70 min", targetType: "server", targetId: host.id, targetName: "pve1", openedAt: iso(ago(4)), lastSeenAt: iso(ago(4)) }).returning())[0].id;
await db.insert(alertIncidents).values({ alertKey: "disk:disk:server:pi:/", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "pi disk nearly full", targetType: "server", targetId: pi.id, targetName: "pi", openedAt: iso(ago(10)), lastSeenAt: iso(ago(10)) });
const audit = (ms: number, category: string, action: string, targetType: string | null, targetId: number | null, detail: unknown = {}) =>
db.insert(auditLog).values({ actorLabel: "Olga", category, action, targetType, targetId: targetId === null ? null : String(targetId), detail: JSON.stringify(detail), createdAt: sqlite(ms) });
await audit(ago(100), "settings", "update", "settings", null, { sections: ["healthChecks"], changes: { healthChecks: { serverOfflineMinutes: { from: 60, to: 30 } } } });
await audit(ago(30), "dns", "create", "dns_provider", 1, { name: "wiki.example.lan" });
await audit(ago(3), "server", "went_offline", "server", host.id, { message: "pve1 hasn't reported for 70 min" });
await audit(ago(2), "server", "went_offline", "server", web.id, { message: "web hasn't reported" }); // this incident again
await audit(ago(8), "server", "update", "server", pi.id, {}); // an unrelated server
await audit(ago(7), "session", "login", "user", 1, {});
await audit(ago(6), "note", "save", "server", web.id, {});
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: host.id, startedAt: iso(ago(65)), endsAt: iso(ago(6)), reason: "Kernel update", createdBy: "Olga" });
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: pi.id, startedAt: iso(ago(30)), endsAt: iso(ago(5)), reason: "Unrelated", createdBy: "Olga" });
await db.insert(updateRuns).values({ serverId: web.id, integrationId: 1, projectId: 1, templateId: 1, templateName: "apt upgrade", hostLimit: "web", status: "done", startedBy: "Olga", startedAt: iso(ago(40)), finishedAt: iso(ago(25)), windowIds: "[]" } as never);
for (const [min, v] of [[100, 50], [70, 51], [45, 66], [20, 72], [5, 73]] as const) await db.insert(serverMetrics).values({ serverId: web.id, metric: "disk:/", value: v, recordedAt: iso(ago(min)) });
for (const [min, v] of [[100, 12], [60, 14], [20, 13]] as const) await db.insert(serverMetrics).values({ serverId: web.id, metric: "cpu", value: v, recordedAt: iso(ago(min)) });
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/incidents", incidentsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/incidents`;
});
after(() => app.close());
test("the likeliest are what is closely related and close in time: the host going down, its maintenance ending, the update run on the server itself", async () => {
const report = (await b.whatChangedBefore(incidentId, 120, new Date(OPENED + 60_000)))!;
const titles = report.suspects.map((s) => s.title);
const top = report.suspects.slice(0, 4).map((s) => s.title);
assert.ok(top.some((t) => /Maintenance on pve1 ended/.test(t)), "the host's maintenance ended six minutes before");
assert.ok(top.some((t) => /another alert was raised: pve1 hasn't reported/i.test(t)), "and its alert was raised first");
assert.ok(top.some((t) => /Update run finished on web: apt upgrade/.test(t)));
assert.ok(titles.some((t) => /Disk \/ rose from 50% to 73%/.test(t)), "a step in the server's own disk");
assert.ok(!titles.some((t) => /CPU load/.test(t)), "a flat figure is nothing");
assert.ok(!titles.some((t) => /Stopped reporting/.test(t)), "the host going down is there once, as the alert, not twice");
const host = report.suspects.find((s) => /Maintenance on pve1 ended/.test(s.title))!;
assert.deepEqual([host.relation, host.confidence, host.kind, host.minutesBefore], ["needs", "likely", "maintenance", 6]);
assert.match(host.why, /something it needs directly, 6 min before/);
assert.equal(report.suspects.find((s) => s.incidentId === hostIncidentId)!.kind, "alert");
for (let i = 1; i < report.suspects.length; i++) assert.ok(report.suspects[i - 1].score >= report.suspects[i].score, "ranked");
});
test("what has nothing to do with it is left out: the other server's update, the login, the note, this incident's own offline record, an unrelated server's maintenance", async () => {
const titles = (await b.whatChangedBefore(incidentId, 120, new Date(OPENED + 60_000)))!.suspects.map((s) => s.title).join("\n");
assert.ok(!/Unrelated/.test(titles), "maintenance on another server");
assert.ok(!/Details changed/.test(titles), "a change to the other server");
assert.ok(!/hasn't reported for 61/.test(titles) && !/Stopped reporting/.test(titles.split("\n").filter((t) => /web/.test(t)).join("")), "not the incident itself");
assert.ok(!/signed in|login/i.test(titles) && !/note/i.test(titles));
});
test("a setting that was changed before is a suspect, and one that bears on this kind of alert counts for more; the further back the window, the more it finds", async () => {
const short = (await b.whatChangedBefore(incidentId, 30, new Date(OPENED + 60_000)))!;
assert.ok(!short.suspects.some((s) => s.kind === "settings"), "100 minutes back is outside 30");
const wide = (await b.whatChangedBefore(incidentId, 240, new Date(OPENED + 60_000)))!;
const setting = wide.suspects.find((s) => s.kind === "settings");
assert.ok(setting, "inside four hours");
assert.equal(setting!.relation, "everything");
assert.ok(setting!.confidence === "background" || setting!.confidence === "possible");
assert.ok(wide.looked.auditEntries >= 6 && wide.looked.maintenanceWindows === 1 && wide.looked.updateRuns === 1 && wide.looked.otherAlerts === 2 && wide.looked.series === 2, JSON.stringify(wide.looked));
});
test("an incident about something with nothing around it says so, with what it looked at", async () => {
const lonely = (await db.insert(alertIncidents).values({ alertKey: "expiry:secret:9", severity: "warning", peakSeverity: "warning", category: "expiry", source: "Secrets", message: "A secret expires", openedAt: iso(OPENED + 5 * 3600_000), lastSeenAt: iso(OPENED + 5 * 3600_000) }).returning())[0];
const report = (await b.whatChangedBefore(lonely.id, 60, new Date(OPENED + 6 * 3600_000)))!;
assert.deepEqual(report.suspects, []);
assert.equal(report.looked.otherAlerts, 0);
});
test("a dependency makes a neighbour: what a server needs counts, what it does not need is nothing", async () => {
const [svc] = await db.select().from(servers).where((await import("drizzle-orm")).eq(servers.name, "pi"));
await db.insert(dependencies).values({ fromType: "server", fromId: webId, toType: "server", toId: svc.id });
const report = (await b.whatChangedBefore(incidentId, 120, new Date(OPENED + 60_000)))!;
const pi = report.suspects.find((s) => /pi disk nearly full/.test(s.title))!;
assert.ok(pi, "pi is now something web needs");
assert.equal(pi.relation, "needs");
});
describe("the address", () => {
test("everyone signed in can ask; the window is read, an unknown incident is 404, and a bad id 400", async () => {
who = "viewer";
const ok = await fetch(`${base}/${incidentId}/before?minutes=45`);
assert.equal(ok.status, 200);
const body = (await ok.json()) as any;
assert.deepEqual([body.incidentId, body.windowMinutes], [incidentId, 45]);
assert.ok(Array.isArray(body.suspects) && body.looked);
assert.equal(((await (await fetch(`${base}/${incidentId}/before?minutes=abc`)).json()) as any).windowMinutes, 120, "an unreadable window is the usual one");
assert.equal(((await (await fetch(`${base}/${incidentId}/before?minutes=99999`)).json()) as any).windowMinutes, 1440);
assert.equal((await fetch(`${base}/999999/before`)).status, 404);
assert.equal((await fetch(`${base}/abc/before`)).status, 400);
});
});
});
+3
View File
@@ -218,6 +218,9 @@ describe("what a token can read", () => {
const incidents = (await (await api("/incidents?status=resolved", secret)).json()) as any;
assert.equal(incidents.incidents.length, 1);
assert.equal(((await (await api(`/incidents/${incidents.incidents[0].id}`, secret)).json()) as any).incident?.id ?? incidents.incidents[0].id, incidents.incidents[0].id);
const before = (await (await api(`/incidents/${incidents.incidents[0].id}/before?minutes=30`, secret)).json()) as any;
assert.deepEqual([before.incidentId, before.windowMinutes, Array.isArray(before.suspects)], [incidents.incidents[0].id, 30, true], "what changed before it, over the same token");
assert.equal((await api("/incidents/999999/before", secret)).status, 404);
const open = ((await (await api("/incidents?status=open", secret)).json()) as any).incidents;
assert.deepEqual(open.map((i: any) => i.alertKey), [`offline:offline:server:${ids.db}`], "the quiet server's alert is an incident that is still open");