/** * Flapping and the noise report: when an alert counts as flapping, what is sent and what is held, how it settles, the flag on the Alerts page, and the report of * which alerts fire most. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { createServer as createHttpServer, type Server } from "node:http"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-flapping-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, heartbeats, alertIncidents, auditLog } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { registerRaise, registerRecovery, hasSettled, flappingKeys, FLAP_WINDOW_MS, SETTLE_MS, keyOfAlertId } = await import("../src/services/flapGuard.js"); const { settleFlappingAlerts } = await import("../src/services/flapMonitor.js"); const { notifyHealthIssues, notifyHealthRecovered } = await import("../src/services/notify.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { buildNoiseReport, mostWithin, hintFor } = await import("../src/services/alertNoise.js"); const { alertsRouter } = await import("../src/routes/alerts.js"); const { updateSettings, setInternalFlag } = await import("../src/services/settingsStore.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); const MIN = 60_000; const entry = (extra: Partial[0]> = {}) => ({ raises: [], flapping: false, since: null, lastEvent: 0, message: "m", ...extra }); describe("when an alert counts as flapping", () => { test("raised the threshold number of times within an hour: the third is the one that is called flapping; after that, raises are held", () => { let e = registerRaise(undefined, 0, 3, "disk"); assert.equal(e.verdict, "pass"); e = registerRaise(e.entry, 10 * MIN, 3, "disk"); assert.equal(e.verdict, "pass"); e = registerRaise(e.entry, 20 * MIN, 3, "disk"); assert.equal(e.verdict, "started"); assert.deepEqual([e.entry.flapping, e.entry.since, e.entry.raises.length], [true, 20 * MIN, 3]); e = registerRaise(e.entry, 25 * MIN, 3, "disk"); assert.equal(e.verdict, "held"); assert.equal(e.entry.flapping, true); }); test("raises more than an hour apart never add up; a threshold of 0 turns it off", () => { let e = registerRaise(undefined, 0, 3, "x"); e = registerRaise(e.entry, FLAP_WINDOW_MS + MIN, 3, "x"); e = registerRaise(e.entry, 2 * FLAP_WINDOW_MS + 2 * MIN, 3, "x"); assert.equal(e.verdict, "pass"); assert.equal(e.entry.raises.length, 1, "only the latest is within the window"); let off = registerRaise(undefined, 0, 0, "x"); for (let i = 1; i < 10; i++) off = registerRaise(off.entry, i * MIN, 0, "x"); assert.equal(off.verdict, "pass"); }); test("a recovery is held only while the alert is flapping, and it counts as a moment for the quiet to be measured from", () => { assert.equal(registerRecovery(undefined, 5).hold, false); assert.equal(registerRecovery(entry({ raises: [1], flapping: false }), 5).hold, false); const held = registerRecovery(entry({ flapping: true, lastEvent: 1 }), 50); assert.deepEqual([held.hold, held.entry?.lastEvent], [true, 50]); }); test("it has settled after an hour with nothing", () => { const e = entry({ flapping: true, lastEvent: 0 }); assert.equal(hasSettled(e, SETTLE_MS - 1), false); assert.equal(hasSettled(e, SETTLE_MS), true); assert.equal(hasSettled(entry({ flapping: false, lastEvent: 0 }), 10 * SETTLE_MS), false); }); test("an alert's id says which condition it is about", () => { assert.equal(keyOfAlertId("disk:disk:server:3:/", "disk"), "disk:server:3:/"); assert.equal(keyOfAlertId("automation:heartbeat:7", "automation"), "heartbeat:7"); }); }); describe("what is sent, what is held, and how it settles", () => { let hook: Server; const messages: string[] = []; const titles = () => messages.map((m) => m.split("\n")[0]); before(async () => { hook = createHttpServer((req, res) => { const chunks: Buffer[] = []; req.on("data", (c) => chunks.push(c)); req.on("end", () => { messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content)); res.end("{}"); }); }); await new Promise((r) => hook.listen(0, "127.0.0.1", r)); await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } }); }); after(() => hook.close()); const issue = { key: "heartbeat:1", message: "Nightly backup hasn't reported", source: "heartbeat", severity: "warning" }; test("raised and cleared again and again: two pairs go out as they are, the third raise is one 'flapping' message, and then it is quiet", async () => { messages.length = 0; await notifyHealthIssues([issue]); await notifyHealthRecovered([issue]); await notifyHealthIssues([issue]); await notifyHealthRecovered([issue]); assert.equal(messages.length, 4, "the first two times are as they always were"); await notifyHealthIssues([issue]); assert.equal(messages.length, 5); assert.match(messages[4], /^\*\*Homelab Manager — Flapping alert/, "titles go out in bold"); assert.match(messages[4], /↻ Nightly backup hasn't reported — raised 3 times within an hour/); assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]); await notifyHealthRecovered([issue]); await notifyHealthIssues([issue]); await notifyHealthRecovered([issue]); assert.equal(messages.length, 5, "nothing more while it flaps"); const audit = await db.select().from(auditLog).where(eq(auditLog.action, "flapping_started")); assert.equal(audit.length, 1); assert.equal(JSON.parse(audit[0].detail!).key, "heartbeat:1"); }); test("another alert is not affected by it, and one without a key is never held", async () => { messages.length = 0; await notifyHealthIssues([{ key: "disk:server:3:/", message: "nas: / is 96% full", severity: "warning" }, { message: "something with no key" }]); assert.equal(messages.length, 1); assert.match(messages[0], /nas: \/ is 96% full/); assert.match(messages[0], /something with no key/); }); test("the Alerts page marks it as flapping while it is, and not after", async () => { const [h] = await db.insert(heartbeats).values({ name: "Nightly backup", token: "hb_x".padEnd(24, "x"), periodMinutes: 60, graceMinutes: 5, createdAt: "2026-01-01 00:00:00" }).returning(); // the real heartbeat's key is heartbeat:: make the held alert that one await setInternalFlag("flapState", JSON.stringify({ [`heartbeat:${h.id}`]: { raises: [Date.now()], flapping: true, since: Date.now(), lastEvent: Date.now(), message: "m" } })); invalidateAlertsCache(); const alert = (await getAlerts(true)).alerts.find((a) => a.id === `automation:heartbeat:${h.id}`)!; assert.ok(alert, "the heartbeat is late, so it is an alert"); assert.equal(alert.flapping, true); await setInternalFlag("flapState", "{}"); invalidateAlertsCache(); assert.equal((await getAlerts(true)).alerts.find((a) => a.id === `automation:heartbeat:${h.id}`)!.flapping, undefined); await db.delete(heartbeats); invalidateAlertsCache(); // the heartbeat that was an alert a moment ago is gone }); test("an hour of quiet settles it: one message says what state it is in; after that it is announced as usual", async () => { await setInternalFlag("flapState", "{}"); messages.length = 0; for (let i = 0; i < 3; i++) { await notifyHealthIssues([issue]); await notifyHealthRecovered([issue]); } assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]); const sent = messages.length; assert.equal(await settleFlappingAlerts(new Date(Date.now() + 30 * MIN)), 0, "half an hour is not enough"); assert.equal(await settleFlappingAlerts(new Date(Date.now() + SETTLE_MS + MIN)), 1); assert.equal(messages.length, sent + 1); assert.match(messages[messages.length - 1], /Alert settled[\s\S]*✓ cleared — Nightly backup hasn't reported/, "nothing says it is still raised, so it says it cleared"); assert.deepEqual([...(await flappingKeys())], []); assert.equal((await db.select().from(auditLog).where(eq(auditLog.action, "flapping_ended"))).length, 1); const before = messages.length; await notifyHealthIssues([issue]); assert.equal(messages.length, before + 1); assert.match(messages[messages.length - 1], /Health Alert/); }); test("turned off, every raise and every recovery is sent", async () => { await setInternalFlag("flapState", "{}"); await updateSettings({ healthChecks: { flapRaises: 0 } }); messages.length = 0; for (let i = 0; i < 5; i++) { await notifyHealthIssues([issue]); await notifyHealthRecovered([issue]); } assert.equal(messages.length, 10); assert.equal(titles().some((t) => t.includes("Flapping")), false); await updateSettings({ healthChecks: { flapRaises: 3 } }); }); test("with the notification switched off, it is still tracked, so switching it on does not start from nothing", async () => { await setInternalFlag("flapState", "{}"); await updateSettings({ notifications: { healthAlerts: false } }); messages.length = 0; for (let i = 0; i < 3; i++) await notifyHealthIssues([issue]); assert.equal(messages.length, 0); assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]); await updateSettings({ notifications: { healthAlerts: true } }); await setInternalFlag("flapState", "{}"); }); }); describe("the noise report", () => { let app: Server; let base = ""; const incident = (alertKey: string, extra: Partial = {}) => ({ alertKey, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: alertKey, openedAt: "2026-10-10T00:00:00.000Z", lastSeenAt: "2026-10-10T00:00:00.000Z", ...extra }); const at = (day: number, hour: number, min = 0) => new Date(Date.UTC(2026, 9, day, hour, min)).toISOString(); const NOW = new Date("2026-10-19T12:00:00Z"); before(async () => { await db.insert(users).values({ oidcSub: "viewer", name: "Vic", role: "viewer" }); await db.delete(alertIncidents); await db.insert(alertIncidents).values([ // a disk that came and went five times, three of them within forty minutes ...[[10, 1, 0, 5], [10, 1, 15, 20], [10, 1, 35, 40], [12, 8, 0, 30], [15, 8, 0, 10]].map(([d, h, m, len]) => incident("disk:disk:server:3:/", { message: "nas: / is 96% full", targetName: "nas", openedAt: at(d, h, m), resolvedAt: new Date(Date.parse(at(d, h, m)) + len * MIN).toISOString(), link: "/servers/3" })), incident("expiry:secret:1", { category: "expiry", message: "Secret expires", openedAt: at(11, 8) }), incident("offline:offline:server:2", { category: "offline", message: "elsa stopped reporting", openedAt: at(5, 8), resolvedAt: at(5, 9) }), incident("offline:offline:server:2", { category: "offline", message: "elsa stopped reporting", openedAt: at(18, 8) }), // before the period incident("disk:disk:server:3:/", { message: "old", openedAt: "2026-05-01T00:00:00.000Z", resolvedAt: "2026-05-01T01:00:00.000Z" }), ]); const a = express(); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: "viewer" } }; next(); }); a.use("/api/alerts", alertsRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/alerts`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("the most times within any hour, however the times are spread", () => { const h = 3600_000; assert.equal(mostWithin([], h), 0); assert.equal(mostWithin([0, 10 * MIN, 20 * MIN, 5 * h], h), 3); assert.equal(mostWithin([0, h, 2 * h], h), 1, "exactly an hour apart is not within an hour"); assert.equal(mostWithin([5 * h, 0, 1 * MIN], h), 2, "in any order"); }); test("what fired most, how long it lasted, which flapped; one that fired once is not noise; an incident before the period is not counted", async () => { const r = await buildNoiseReport(30, NOW); assert.deepEqual([r.days, r.incidents, r.distinctAlerts, r.repeating, r.flapped], [30, 8, 3, 2, 1]); const disk = r.rows[0]; assert.deepEqual([disk.alertId, disk.fired, disk.totalMinutes, disk.averageMinutes, disk.longestMinutes, disk.mostInAnHour, disk.flapped, disk.openNow], ["disk:disk:server:3:/", 5, 105, 21, 40, 3, true, false]); assert.equal(disk.message, "nas: / is 96% full", "what it said last"); assert.equal(disk.where, "nas"); assert.match(disk.hint, /Raise the limit for this disk/); const elsa = r.rows[1]; assert.deepEqual([elsa.fired, elsa.openNow, elsa.flapped], [2, true, false]); assert.match(elsa.hint, /Server offline after/); assert.equal(elsa.longestMinutes > 60, true, "the open one has been going since it began"); assert.equal(r.rows.some((x) => x.alertId === "expiry:secret:1"), false); assert.deepEqual(r.byCategory.map((c) => [c.category, c.incidents, c.repeating]), [["disk", 5, 1], ["offline", 2, 1], ["expiry", 1, 0]]); assert.equal((await buildNoiseReport(180, NOW)).incidents, 9, "a longer period reaches the old one"); assert.equal((await buildNoiseReport(7, NOW)).incidents, 2, "a shorter one leaves most out"); }); test("an alert that is flapping now says so", async () => { await setInternalFlag("flapState", JSON.stringify({ "disk:server:3:/": { raises: [Date.now()], flapping: true, since: Date.now(), lastEvent: Date.now(), message: "m" } })); const r = await buildNoiseReport(30, NOW); assert.equal(r.rows[0].flappingNow, true); assert.equal(r.rows[1].flappingNow, false); await setInternalFlag("flapState", "{}"); }); test("advice for what the alert is about", () => { assert.match(hintFor("monitoring", "monitoring:new-port:server:1:tcp:8080"), /Ports card/); assert.match(hintFor("automation", "automation:heartbeat:1"), /grace/); assert.match(hintFor("monitoring", "monitoring:endpoint:2"), /failures in a row/); assert.match(hintFor("services", "services:failed-units:server:1"), /Services to ignore/); assert.match(hintFor("tickets", "tickets:x"), /Acknowledge it/); }); test("everyone signed in can read it; the period is one of four, or thirty", async () => { const body = await (await fetch(`${base}/noise?days=90`)).json(); assert.equal(body.days, 90); assert.equal((await (await fetch(`${base}/noise?days=banana`)).json()).days, 30); assert.equal((await (await fetch(`${base}/noise?days=45`)).json()).days, 30, "not an arbitrary number"); assert.ok(Array.isArray(body.rows)); }); });