Flapping alerts, a noise report and bulk actions with undo

Flapping: an alert raised three times within an hour is announced once as
flapping, its further raise/clear messages are held until it has been calm
for an hour, and one message then says whether it is still active. Event
webhooks still hear everything. The state survives restarts.

Alerts > Noise lists the alerts that keep coming back (7-180 days): how often,
how long, most in an hour, and a hint on what helps.

Bulk actions on Alerts (acknowledge, remove acknowledgement, silence), Servers
(tag, silence, runs on) and Inventory (location and dates, print labels). Each
reports what it left alone and why, and has an Undo on the toast.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 23:55:09 +02:00
1 parent be6c5552bd
commit 73a06ebe63
35 files changed
+2546 -17

No files matched your search

+287
View File
@@ -0,0 +1,287 @@
/**
* Flapping and the noise report: when an alert counts as flapping, what is sent and what is held, how it settles, the flag on the Alerts page, and the report of
* which alerts fire most.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-flapping-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, heartbeats, alertIncidents, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { registerRaise, registerRecovery, hasSettled, flappingKeys, FLAP_WINDOW_MS, SETTLE_MS, keyOfAlertId } = await import("../src/services/flapGuard.js");
const { settleFlappingAlerts } = await import("../src/services/flapMonitor.js");
const { notifyHealthIssues, notifyHealthRecovered } = await import("../src/services/notify.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { buildNoiseReport, mostWithin, hintFor } = await import("../src/services/alertNoise.js");
const { alertsRouter } = await import("../src/routes/alerts.js");
const { updateSettings, setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const entry = (extra: Partial<Parameters<typeof hasSettled>[0]> = {}) => ({ raises: [], flapping: false, since: null, lastEvent: 0, message: "m", ...extra });
describe("when an alert counts as flapping", () => {
test("raised the threshold number of times within an hour: the third is the one that is called flapping; after that, raises are held", () => {
let e = registerRaise(undefined, 0, 3, "disk");
assert.equal(e.verdict, "pass");
e = registerRaise(e.entry, 10 * MIN, 3, "disk");
assert.equal(e.verdict, "pass");
e = registerRaise(e.entry, 20 * MIN, 3, "disk");
assert.equal(e.verdict, "started");
assert.deepEqual([e.entry.flapping, e.entry.since, e.entry.raises.length], [true, 20 * MIN, 3]);
e = registerRaise(e.entry, 25 * MIN, 3, "disk");
assert.equal(e.verdict, "held");
assert.equal(e.entry.flapping, true);
});
test("raises more than an hour apart never add up; a threshold of 0 turns it off", () => {
let e = registerRaise(undefined, 0, 3, "x");
e = registerRaise(e.entry, FLAP_WINDOW_MS + MIN, 3, "x");
e = registerRaise(e.entry, 2 * FLAP_WINDOW_MS + 2 * MIN, 3, "x");
assert.equal(e.verdict, "pass");
assert.equal(e.entry.raises.length, 1, "only the latest is within the window");
let off = registerRaise(undefined, 0, 0, "x");
for (let i = 1; i < 10; i++) off = registerRaise(off.entry, i * MIN, 0, "x");
assert.equal(off.verdict, "pass");
});
test("a recovery is held only while the alert is flapping, and it counts as a moment for the quiet to be measured from", () => {
assert.equal(registerRecovery(undefined, 5).hold, false);
assert.equal(registerRecovery(entry({ raises: [1], flapping: false }), 5).hold, false);
const held = registerRecovery(entry({ flapping: true, lastEvent: 1 }), 50);
assert.deepEqual([held.hold, held.entry?.lastEvent], [true, 50]);
});
test("it has settled after an hour with nothing", () => {
const e = entry({ flapping: true, lastEvent: 0 });
assert.equal(hasSettled(e, SETTLE_MS - 1), false);
assert.equal(hasSettled(e, SETTLE_MS), true);
assert.equal(hasSettled(entry({ flapping: false, lastEvent: 0 }), 10 * SETTLE_MS), false);
});
test("an alert's id says which condition it is about", () => {
assert.equal(keyOfAlertId("disk:disk:server:3:/", "disk"), "disk:server:3:/");
assert.equal(keyOfAlertId("automation:heartbeat:7", "automation"), "heartbeat:7");
});
});
describe("what is sent, what is held, and how it settles", () => {
let hook: Server;
const messages: string[] = [];
const titles = () => messages.map((m) => m.split("\n")[0]);
before(async () => {
hook = createHttpServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c) => chunks.push(c));
req.on("end", () => {
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
res.end("{}");
});
});
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
});
after(() => hook.close());
const issue = { key: "heartbeat:1", message: "Nightly backup hasn't reported", source: "heartbeat", severity: "warning" };
test("raised and cleared again and again: two pairs go out as they are, the third raise is one 'flapping' message, and then it is quiet", async () => {
messages.length = 0;
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
assert.equal(messages.length, 4, "the first two times are as they always were");
await notifyHealthIssues([issue]);
assert.equal(messages.length, 5);
assert.match(messages[4], /^\*\*Homelab Manager — Flapping alert/, "titles go out in bold");
assert.match(messages[4], /↻ Nightly backup hasn't reported — raised 3 times within an hour/);
assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]);
await notifyHealthRecovered([issue]);
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
assert.equal(messages.length, 5, "nothing more while it flaps");
const audit = await db.select().from(auditLog).where(eq(auditLog.action, "flapping_started"));
assert.equal(audit.length, 1);
assert.equal(JSON.parse(audit[0].detail!).key, "heartbeat:1");
});
test("another alert is not affected by it, and one without a key is never held", async () => {
messages.length = 0;
await notifyHealthIssues([{ key: "disk:server:3:/", message: "nas: / is 96% full", severity: "warning" }, { message: "something with no key" }]);
assert.equal(messages.length, 1);
assert.match(messages[0], /nas: \/ is 96% full/);
assert.match(messages[0], /something with no key/);
});
test("the Alerts page marks it as flapping while it is, and not after", async () => {
const [h] = await db.insert(heartbeats).values({ name: "Nightly backup", token: "hb_x".padEnd(24, "x"), periodMinutes: 60, graceMinutes: 5, createdAt: "2026-01-01 00:00:00" }).returning();
// the real heartbeat's key is heartbeat:<id>: make the held alert that one
await setInternalFlag("flapState", JSON.stringify({ [`heartbeat:${h.id}`]: { raises: [Date.now()], flapping: true, since: Date.now(), lastEvent: Date.now(), message: "m" } }));
invalidateAlertsCache();
const alert = (await getAlerts(true)).alerts.find((a) => a.id === `automation:heartbeat:${h.id}`)!;
assert.ok(alert, "the heartbeat is late, so it is an alert");
assert.equal(alert.flapping, true);
await setInternalFlag("flapState", "{}");
invalidateAlertsCache();
assert.equal((await getAlerts(true)).alerts.find((a) => a.id === `automation:heartbeat:${h.id}`)!.flapping, undefined);
await db.delete(heartbeats);
invalidateAlertsCache(); // the heartbeat that was an alert a moment ago is gone
});
test("an hour of quiet settles it: one message says what state it is in; after that it is announced as usual", async () => {
await setInternalFlag("flapState", "{}");
messages.length = 0;
for (let i = 0; i < 3; i++) {
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
}
assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]);
const sent = messages.length;
assert.equal(await settleFlappingAlerts(new Date(Date.now() + 30 * MIN)), 0, "half an hour is not enough");
assert.equal(await settleFlappingAlerts(new Date(Date.now() + SETTLE_MS + MIN)), 1);
assert.equal(messages.length, sent + 1);
assert.match(messages[messages.length - 1], /Alert settled[\s\S]*✓ cleared — Nightly backup hasn't reported/, "nothing says it is still raised, so it says it cleared");
assert.deepEqual([...(await flappingKeys())], []);
assert.equal((await db.select().from(auditLog).where(eq(auditLog.action, "flapping_ended"))).length, 1);
const before = messages.length;
await notifyHealthIssues([issue]);
assert.equal(messages.length, before + 1);
assert.match(messages[messages.length - 1], /Health Alert/);
});
test("turned off, every raise and every recovery is sent", async () => {
await setInternalFlag("flapState", "{}");
await updateSettings({ healthChecks: { flapRaises: 0 } });
messages.length = 0;
for (let i = 0; i < 5; i++) {
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
}
assert.equal(messages.length, 10);
assert.equal(titles().some((t) => t.includes("Flapping")), false);
await updateSettings({ healthChecks: { flapRaises: 3 } });
});
test("with the notification switched off, it is still tracked, so switching it on does not start from nothing", async () => {
await setInternalFlag("flapState", "{}");
await updateSettings({ notifications: { healthAlerts: false } });
messages.length = 0;
for (let i = 0; i < 3; i++) await notifyHealthIssues([issue]);
assert.equal(messages.length, 0);
assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]);
await updateSettings({ notifications: { healthAlerts: true } });
await setInternalFlag("flapState", "{}");
});
});
describe("the noise report", () => {
let app: Server;
let base = "";
const incident = (alertKey: string, extra: Partial<typeof alertIncidents.$inferInsert> = {}) => ({ alertKey, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: alertKey, openedAt: "2026-10-10T00:00:00.000Z", lastSeenAt: "2026-10-10T00:00:00.000Z", ...extra });
const at = (day: number, hour: number, min = 0) => new Date(Date.UTC(2026, 9, day, hour, min)).toISOString();
const NOW = new Date("2026-10-19T12:00:00Z");
before(async () => {
await db.insert(users).values({ oidcSub: "viewer", name: "Vic", role: "viewer" });
await db.delete(alertIncidents);
await db.insert(alertIncidents).values([
// a disk that came and went five times, three of them within forty minutes
...[[10, 1, 0, 5], [10, 1, 15, 20], [10, 1, 35, 40], [12, 8, 0, 30], [15, 8, 0, 10]].map(([d, h, m, len]) => incident("disk:disk:server:3:/", { message: "nas: / is 96% full", targetName: "nas", openedAt: at(d, h, m), resolvedAt: new Date(Date.parse(at(d, h, m)) + len * MIN).toISOString(), link: "/servers/3" })),
incident("expiry:secret:1", { category: "expiry", message: "Secret expires", openedAt: at(11, 8) }),
incident("offline:offline:server:2", { category: "offline", message: "elsa stopped reporting", openedAt: at(5, 8), resolvedAt: at(5, 9) }),
incident("offline:offline:server:2", { category: "offline", message: "elsa stopped reporting", openedAt: at(18, 8) }),
// before the period
incident("disk:disk:server:3:/", { message: "old", openedAt: "2026-05-01T00:00:00.000Z", resolvedAt: "2026-05-01T01:00:00.000Z" }),
]);
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/alerts", alertsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/alerts`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the most times within any hour, however the times are spread", () => {
const h = 3600_000;
assert.equal(mostWithin([], h), 0);
assert.equal(mostWithin([0, 10 * MIN, 20 * MIN, 5 * h], h), 3);
assert.equal(mostWithin([0, h, 2 * h], h), 1, "exactly an hour apart is not within an hour");
assert.equal(mostWithin([5 * h, 0, 1 * MIN], h), 2, "in any order");
});
test("what fired most, how long it lasted, which flapped; one that fired once is not noise; an incident before the period is not counted", async () => {
const r = await buildNoiseReport(30, NOW);
assert.deepEqual([r.days, r.incidents, r.distinctAlerts, r.repeating, r.flapped], [30, 8, 3, 2, 1]);
const disk = r.rows[0];
assert.deepEqual([disk.alertId, disk.fired, disk.totalMinutes, disk.averageMinutes, disk.longestMinutes, disk.mostInAnHour, disk.flapped, disk.openNow], ["disk:disk:server:3:/", 5, 105, 21, 40, 3, true, false]);
assert.equal(disk.message, "nas: / is 96% full", "what it said last");
assert.equal(disk.where, "nas");
assert.match(disk.hint, /Raise the limit for this disk/);
const elsa = r.rows[1];
assert.deepEqual([elsa.fired, elsa.openNow, elsa.flapped], [2, true, false]);
assert.match(elsa.hint, /Server offline after/);
assert.equal(elsa.longestMinutes > 60, true, "the open one has been going since it began");
assert.equal(r.rows.some((x) => x.alertId === "expiry:secret:1"), false);
assert.deepEqual(r.byCategory.map((c) => [c.category, c.incidents, c.repeating]), [["disk", 5, 1], ["offline", 2, 1], ["expiry", 1, 0]]);
assert.equal((await buildNoiseReport(180, NOW)).incidents, 9, "a longer period reaches the old one");
assert.equal((await buildNoiseReport(7, NOW)).incidents, 2, "a shorter one leaves most out");
});
test("an alert that is flapping now says so", async () => {
await setInternalFlag("flapState", JSON.stringify({ "disk:server:3:/": { raises: [Date.now()], flapping: true, since: Date.now(), lastEvent: Date.now(), message: "m" } }));
const r = await buildNoiseReport(30, NOW);
assert.equal(r.rows[0].flappingNow, true);
assert.equal(r.rows[1].flappingNow, false);
await setInternalFlag("flapState", "{}");
});
test("advice for what the alert is about", () => {
assert.match(hintFor("monitoring", "monitoring:new-port:server:1:tcp:8080"), /Ports card/);
assert.match(hintFor("automation", "automation:heartbeat:1"), /grace/);
assert.match(hintFor("monitoring", "monitoring:endpoint:2"), /failures in a row/);
assert.match(hintFor("services", "services:failed-units:server:1"), /Services to ignore/);
assert.match(hintFor("tickets", "tickets:x"), /Acknowledge it/);
});
test("everyone signed in can read it; the period is one of four, or thirty", async () => {
const body = await (await fetch(`${base}/noise?days=90`)).json();
assert.equal(body.days, 90);
assert.equal((await (await fetch(`${base}/noise?days=banana`)).json()).days, 30);
assert.equal((await (await fetch(`${base}/noise?days=45`)).json()).days, 30, "not an arbitrary number");
assert.ok(Array.isArray(body.rows));
});
});