- A card on the incident page, and GET /api/incidents/:id/before and /api/v1/incidents/:id/before: what the app knows happened in the 30 min to 24 h before an alert began (audit log, maintenance windows, update runs and patch plans, other alerts, a step in the server's own figures), scored by how related, what kind and how close - A host going down is shown once; Likely, Possible and Background; at most ten - Docs and roles updated Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
231 lines
16 KiB
TypeScript
231 lines
16 KiB
TypeScript
/**
|
|
* "What changed before this?": ranking what happened in the minutes before an incident, from the audit log, maintenance windows, update runs, other alerts and the
|
|
* server's own figures.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { AddressInfo } from "node:net";
|
|
import type { Server } from "node:http";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-before-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const express = (await import("express")).default;
|
|
const { db } = await import("../src/db/client.js");
|
|
const { users, servers, auditLog, maintenanceWindows, updateRuns, alertIncidents, serverMetrics, dependencies } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const b = await import("../src/services/incidentBefore.js");
|
|
const { incidentsRouter } = await import("../src/routes/incidents.js");
|
|
|
|
await runMigrations();
|
|
|
|
const OPENED = Date.parse("2026-10-17T12:00:00.000Z");
|
|
const ago = (minutes: number) => OPENED - minutes * 60_000;
|
|
const iso = (ms: number) => new Date(ms).toISOString();
|
|
const sqlite = (ms: number) => iso(ms).slice(0, 19).replace("T", " ");
|
|
|
|
const cand = (extra: Partial<Parameters<typeof b.rankSuspects>[0][number]> = {}): Parameters<typeof b.rankSuspects>[0][number] => ({ at: ago(10), kind: "change", title: "x", detail: null, actor: null, relation: "same", distance: 0, aboutName: null, weight: 1, link: null, ...extra });
|
|
|
|
after(() => {
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows keeps the database file locked until the process exits.
|
|
}
|
|
});
|
|
|
|
describe("how a suspect is scored", () => {
|
|
test("close and related beats far and unrelated; a change five minutes before counts for far more than one two hours before", () => {
|
|
assert.ok(b.recency(5) > 0.9 && b.recency(60) < 0.4 && b.recency(120) === 0.25 && b.recency(1000) === 0.25, "never nothing inside the window, never more than 1");
|
|
assert.equal(b.recency(0), 1);
|
|
const r = (relation: Parameters<typeof b.relevance>[0], d = 0) => b.relevance(relation, d);
|
|
assert.ok(r("same") > r("needs", 1) && r("needs", 1) > r("needs", 2) && r("needs", 2) > r("everything") && r("everything") > r("needed_by", 1) && r("needed_by", 1) > r("needed_by", 2) && r("needed_by", 2) > r("other"), "the thing itself, what it needs, a change to everything, what needs it, nothing in common");
|
|
assert.deepEqual([b.confidenceOf(90), b.confidenceOf(40), b.confidenceOf(10)], ["likely", "possible", "background"]);
|
|
});
|
|
|
|
test("they are ranked by score, the negligible are left out, and a tie goes to the nearer", () => {
|
|
const ranked = b.rankSuspects(
|
|
[
|
|
cand({ title: "settings long ago", at: ago(115), relation: "everything", weight: 0.5 }),
|
|
cand({ title: "host maintenance ended", at: ago(5), relation: "needs", distance: 1, weight: 1 }),
|
|
cand({ title: "same, a bit earlier", at: ago(30), relation: "same", weight: 1 }),
|
|
cand({ title: "unrelated and old", at: ago(110), relation: "other", weight: 0.3 }),
|
|
],
|
|
OPENED,
|
|
);
|
|
assert.deepEqual(ranked.map((s) => s.title), ["host maintenance ended", "same, a bit earlier", "settings long ago"], "the unrelated old one scores under the floor");
|
|
assert.equal(ranked[0].confidence, "likely");
|
|
assert.match(ranked[0].why, /About something it needs directly, 5 min before/);
|
|
assert.equal(ranked[0].about, "something it needs — which it needs");
|
|
assert.equal(ranked[2].confidence, "background");
|
|
assert.equal(b.rankSuspects(Array.from({ length: 30 }, (_, i) => cand({ title: `c${i}`, at: ago(1 + i) })), OPENED).length, b.MAX_SUSPECTS);
|
|
});
|
|
|
|
test("a step in a series needs three readings and a real move, and is dated by the biggest single move", () => {
|
|
const series = (...v: number[]) => v.map((x, i) => ({ t: ago(60 - i * 10), v: x }));
|
|
assert.equal(b.stepOf(series(50, 71), 8), null, "two readings say nothing");
|
|
assert.equal(b.stepOf(series(50, 52, 53), 8), null, "a small move");
|
|
const step = b.stepOf(series(50, 51, 66, 71), 8)!;
|
|
assert.deepEqual([step.from, step.to, step.at], [50, 71, ago(40)]);
|
|
assert.equal(b.stepOf(series(80, 60, 40), 8)!.to, 40, "falling counts too");
|
|
});
|
|
|
|
test("what an audit entry is worth: a login or a note is nothing; a host going down is a lot; a setting that bears on this alert is more than one that doesn't", () => {
|
|
const ctx = { sameThing: false, neighbour: false, incidentCategory: "offline", changedSections: [] as string[] };
|
|
assert.equal(b.auditWeight("session", "login", ctx), null);
|
|
assert.equal(b.auditWeight("note", "save", ctx), null);
|
|
assert.equal(b.auditWeight("runbook", "create", ctx), null);
|
|
assert.equal(b.auditWeight("server", "went_offline", { ...ctx, sameThing: true }), null, "for the thing itself that is only the incident again");
|
|
assert.equal(b.auditWeight("server", "went_offline", { ...ctx, neighbour: true })!.weight, 0.95);
|
|
assert.ok(b.auditWeight("settings", "update", { ...ctx, changedSections: ["healthChecks"] })!.weight > b.auditWeight("settings", "update", { ...ctx, changedSections: ["display"] })!.weight);
|
|
assert.equal(b.auditWeight("settings", "update", { ...ctx, incidentCategory: "expiry", changedSections: ["healthChecks"] })!.weight, 0.5, "not relevant to an expiry");
|
|
assert.equal(b.auditWeight("integration", "update", ctx), null, "an unrelated integration");
|
|
assert.equal(b.auditWeight("integration", "update", { ...ctx, neighbour: true })!.weight, 0.8);
|
|
});
|
|
|
|
test("a host going down is told once: the alert about it is kept and the audit entry for the same moment is dropped", () => {
|
|
const entry = cand({ title: "Stopped reporting", at: ago(3), eventKey: "server:4:offline" });
|
|
const alert = cand({ title: "Another alert was raised", at: ago(4), eventKey: "server:4:offline", incidentId: 7 });
|
|
assert.deepEqual(b.withoutRepeats([entry, alert]).map((c) => c.title), ["Another alert was raised"]);
|
|
assert.equal(b.withoutRepeats([entry]).length, 1, "with no alert to stand for it, the entry stays");
|
|
assert.equal(b.withoutRepeats([entry, { ...alert, at: ago(30) }]).length, 2, "an alert half an hour apart is a different event");
|
|
assert.equal(b.withoutRepeats([entry, { ...alert, eventKey: "server:5:offline" }]).length, 2, "and one about another server too");
|
|
});
|
|
|
|
test("the window is kept to something sensible", () => {
|
|
assert.deepEqual([b.clampMinutes(undefined), b.clampMinutes("abc"), b.clampMinutes(1), b.clampMinutes(99999), b.clampMinutes(45)], [120, 120, 15, 1440, 45]);
|
|
});
|
|
});
|
|
|
|
describe("reading what happened", () => {
|
|
let app: Server;
|
|
let base = "";
|
|
let who = "viewer";
|
|
let webId = 0;
|
|
let incidentId = 0;
|
|
let hostIncidentId = 0;
|
|
|
|
before(async () => {
|
|
await db.insert(users).values([
|
|
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
|
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
|
]);
|
|
const mk = async (name: string, extra: Record<string, unknown> = {}) => (await db.insert(servers).values({ name, apiTokenHash: `h-${name}`, apiTokenPrefix: name.slice(0, 3), ipAddresses: "[]", ...extra } as never).returning())[0];
|
|
const host = await mk("pve1");
|
|
const web = await mk("web", { parentServerId: host.id });
|
|
const pi = await mk("pi");
|
|
webId = web.id;
|
|
|
|
// The incident: web went wrong, five minutes after its host did.
|
|
const inc = (await db.insert(alertIncidents).values({ alertKey: `disk:disk:server:${web.id}:/`, severity: "warning", peakSeverity: "warning", category: "offline", source: "Server", message: "web hasn't reported for 61 min", targetType: "server", targetId: web.id, targetName: "web", openedAt: iso(OPENED), lastSeenAt: iso(OPENED) }).returning())[0];
|
|
incidentId = inc.id;
|
|
hostIncidentId = (await db.insert(alertIncidents).values({ alertKey: `offline:offline:server:${host.id}`, severity: "critical", peakSeverity: "critical", category: "offline", source: "Server", message: "pve1 hasn't reported for 70 min", targetType: "server", targetId: host.id, targetName: "pve1", openedAt: iso(ago(4)), lastSeenAt: iso(ago(4)) }).returning())[0].id;
|
|
await db.insert(alertIncidents).values({ alertKey: "disk:disk:server:pi:/", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "pi disk nearly full", targetType: "server", targetId: pi.id, targetName: "pi", openedAt: iso(ago(10)), lastSeenAt: iso(ago(10)) });
|
|
|
|
const audit = (ms: number, category: string, action: string, targetType: string | null, targetId: number | null, detail: unknown = {}) =>
|
|
db.insert(auditLog).values({ actorLabel: "Olga", category, action, targetType, targetId: targetId === null ? null : String(targetId), detail: JSON.stringify(detail), createdAt: sqlite(ms) });
|
|
await audit(ago(100), "settings", "update", "settings", null, { sections: ["healthChecks"], changes: { healthChecks: { serverOfflineMinutes: { from: 60, to: 30 } } } });
|
|
await audit(ago(30), "dns", "create", "dns_provider", 1, { name: "wiki.example.lan" });
|
|
await audit(ago(3), "server", "went_offline", "server", host.id, { message: "pve1 hasn't reported for 70 min" });
|
|
await audit(ago(2), "server", "went_offline", "server", web.id, { message: "web hasn't reported" }); // this incident again
|
|
await audit(ago(8), "server", "update", "server", pi.id, {}); // an unrelated server
|
|
await audit(ago(7), "session", "login", "user", 1, {});
|
|
await audit(ago(6), "note", "save", "server", web.id, {});
|
|
|
|
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: host.id, startedAt: iso(ago(65)), endsAt: iso(ago(6)), reason: "Kernel update", createdBy: "Olga" });
|
|
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: pi.id, startedAt: iso(ago(30)), endsAt: iso(ago(5)), reason: "Unrelated", createdBy: "Olga" });
|
|
await db.insert(updateRuns).values({ serverId: web.id, integrationId: 1, projectId: 1, templateId: 1, templateName: "apt upgrade", hostLimit: "web", status: "done", startedBy: "Olga", startedAt: iso(ago(40)), finishedAt: iso(ago(25)), windowIds: "[]" } as never);
|
|
|
|
for (const [min, v] of [[100, 50], [70, 51], [45, 66], [20, 72], [5, 73]] as const) await db.insert(serverMetrics).values({ serverId: web.id, metric: "disk:/", value: v, recordedAt: iso(ago(min)) });
|
|
for (const [min, v] of [[100, 12], [60, 14], [20, 13]] as const) await db.insert(serverMetrics).values({ serverId: web.id, metric: "cpu", value: v, recordedAt: iso(ago(min)) });
|
|
|
|
const a = express();
|
|
a.use(express.json());
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
|
next();
|
|
});
|
|
a.use("/api/incidents", incidentsRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/incidents`;
|
|
});
|
|
after(() => app.close());
|
|
|
|
test("the likeliest are what is closely related and close in time: the host going down, its maintenance ending, the update run on the server itself", async () => {
|
|
const report = (await b.whatChangedBefore(incidentId, 120, new Date(OPENED + 60_000)))!;
|
|
const titles = report.suspects.map((s) => s.title);
|
|
const top = report.suspects.slice(0, 4).map((s) => s.title);
|
|
assert.ok(top.some((t) => /Maintenance on pve1 ended/.test(t)), "the host's maintenance ended six minutes before");
|
|
assert.ok(top.some((t) => /another alert was raised: pve1 hasn't reported/i.test(t)), "and its alert was raised first");
|
|
assert.ok(top.some((t) => /Update run finished on web: apt upgrade/.test(t)));
|
|
assert.ok(titles.some((t) => /Disk \/ rose from 50% to 73%/.test(t)), "a step in the server's own disk");
|
|
assert.ok(!titles.some((t) => /CPU load/.test(t)), "a flat figure is nothing");
|
|
assert.ok(!titles.some((t) => /Stopped reporting/.test(t)), "the host going down is there once, as the alert, not twice");
|
|
const host = report.suspects.find((s) => /Maintenance on pve1 ended/.test(s.title))!;
|
|
assert.deepEqual([host.relation, host.confidence, host.kind, host.minutesBefore], ["needs", "likely", "maintenance", 6]);
|
|
assert.match(host.why, /something it needs directly, 6 min before/);
|
|
assert.equal(report.suspects.find((s) => s.incidentId === hostIncidentId)!.kind, "alert");
|
|
for (let i = 1; i < report.suspects.length; i++) assert.ok(report.suspects[i - 1].score >= report.suspects[i].score, "ranked");
|
|
});
|
|
|
|
test("what has nothing to do with it is left out: the other server's update, the login, the note, this incident's own offline record, an unrelated server's maintenance", async () => {
|
|
const titles = (await b.whatChangedBefore(incidentId, 120, new Date(OPENED + 60_000)))!.suspects.map((s) => s.title).join("\n");
|
|
assert.ok(!/Unrelated/.test(titles), "maintenance on another server");
|
|
assert.ok(!/Details changed/.test(titles), "a change to the other server");
|
|
assert.ok(!/hasn't reported for 61/.test(titles) && !/Stopped reporting/.test(titles.split("\n").filter((t) => /web/.test(t)).join("")), "not the incident itself");
|
|
assert.ok(!/signed in|login/i.test(titles) && !/note/i.test(titles));
|
|
});
|
|
|
|
test("a setting that was changed before is a suspect, and one that bears on this kind of alert counts for more; the further back the window, the more it finds", async () => {
|
|
const short = (await b.whatChangedBefore(incidentId, 30, new Date(OPENED + 60_000)))!;
|
|
assert.ok(!short.suspects.some((s) => s.kind === "settings"), "100 minutes back is outside 30");
|
|
const wide = (await b.whatChangedBefore(incidentId, 240, new Date(OPENED + 60_000)))!;
|
|
const setting = wide.suspects.find((s) => s.kind === "settings");
|
|
assert.ok(setting, "inside four hours");
|
|
assert.equal(setting!.relation, "everything");
|
|
assert.ok(setting!.confidence === "background" || setting!.confidence === "possible");
|
|
assert.ok(wide.looked.auditEntries >= 6 && wide.looked.maintenanceWindows === 1 && wide.looked.updateRuns === 1 && wide.looked.otherAlerts === 2 && wide.looked.series === 2, JSON.stringify(wide.looked));
|
|
});
|
|
|
|
test("an incident about something with nothing around it says so, with what it looked at", async () => {
|
|
const lonely = (await db.insert(alertIncidents).values({ alertKey: "expiry:secret:9", severity: "warning", peakSeverity: "warning", category: "expiry", source: "Secrets", message: "A secret expires", openedAt: iso(OPENED + 5 * 3600_000), lastSeenAt: iso(OPENED + 5 * 3600_000) }).returning())[0];
|
|
const report = (await b.whatChangedBefore(lonely.id, 60, new Date(OPENED + 6 * 3600_000)))!;
|
|
assert.deepEqual(report.suspects, []);
|
|
assert.equal(report.looked.otherAlerts, 0);
|
|
});
|
|
|
|
test("a dependency makes a neighbour: what a server needs counts, what it does not need is nothing", async () => {
|
|
const [svc] = await db.select().from(servers).where((await import("drizzle-orm")).eq(servers.name, "pi"));
|
|
await db.insert(dependencies).values({ fromType: "server", fromId: webId, toType: "server", toId: svc.id });
|
|
const report = (await b.whatChangedBefore(incidentId, 120, new Date(OPENED + 60_000)))!;
|
|
const pi = report.suspects.find((s) => /pi disk nearly full/.test(s.title))!;
|
|
assert.ok(pi, "pi is now something web needs");
|
|
assert.equal(pi.relation, "needs");
|
|
});
|
|
|
|
describe("the address", () => {
|
|
test("everyone signed in can ask; the window is read, an unknown incident is 404, and a bad id 400", async () => {
|
|
who = "viewer";
|
|
const ok = await fetch(`${base}/${incidentId}/before?minutes=45`);
|
|
assert.equal(ok.status, 200);
|
|
const body = (await ok.json()) as any;
|
|
assert.deepEqual([body.incidentId, body.windowMinutes], [incidentId, 45]);
|
|
assert.ok(Array.isArray(body.suspects) && body.looked);
|
|
assert.equal(((await (await fetch(`${base}/${incidentId}/before?minutes=abc`)).json()) as any).windowMinutes, 120, "an unreadable window is the usual one");
|
|
assert.equal(((await (await fetch(`${base}/${incidentId}/before?minutes=99999`)).json()) as any).windowMinutes, 1440);
|
|
assert.equal((await fetch(`${base}/999999/before`)).status, 404);
|
|
assert.equal((await fetch(`${base}/abc/before`)).status, 400);
|
|
});
|
|
});
|
|
});
|