Files
bobbanandClaude Sonnet 5.5 06ac120a1d Services: opt-in monitoring with a per-server watch list
A failed service now raises an alert (and counts in /metrics and the public API) only when the server is told to watch it, by name or pattern, or * for all. Failed services that are not watched still show on the Host health card with a Watch button. Replaces the ignore list.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-16 21:48:32 +01:00

288 lines
15 KiB
TypeScript

/**
* Flapping and the noise report: when an alert counts as flapping, what is sent and what is held, how it settles, the flag on the Alerts page, and the report of
* which alerts fire most.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-flapping-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, heartbeats, alertIncidents, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { registerRaise, registerRecovery, hasSettled, flappingKeys, FLAP_WINDOW_MS, SETTLE_MS, keyOfAlertId } = await import("../src/services/flapGuard.js");
const { settleFlappingAlerts } = await import("../src/services/flapMonitor.js");
const { notifyHealthIssues, notifyHealthRecovered } = await import("../src/services/notify.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { buildNoiseReport, mostWithin, hintFor } = await import("../src/services/alertNoise.js");
const { alertsRouter } = await import("../src/routes/alerts.js");
const { updateSettings, setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const entry = (extra: Partial<Parameters<typeof hasSettled>[0]> = {}) => ({ raises: [], flapping: false, since: null, lastEvent: 0, message: "m", ...extra });
describe("when an alert counts as flapping", () => {
test("raised the threshold number of times within an hour: the third is the one that is called flapping; after that, raises are held", () => {
let e = registerRaise(undefined, 0, 3, "disk");
assert.equal(e.verdict, "pass");
e = registerRaise(e.entry, 10 * MIN, 3, "disk");
assert.equal(e.verdict, "pass");
e = registerRaise(e.entry, 20 * MIN, 3, "disk");
assert.equal(e.verdict, "started");
assert.deepEqual([e.entry.flapping, e.entry.since, e.entry.raises.length], [true, 20 * MIN, 3]);
e = registerRaise(e.entry, 25 * MIN, 3, "disk");
assert.equal(e.verdict, "held");
assert.equal(e.entry.flapping, true);
});
test("raises more than an hour apart never add up; a threshold of 0 turns it off", () => {
let e = registerRaise(undefined, 0, 3, "x");
e = registerRaise(e.entry, FLAP_WINDOW_MS + MIN, 3, "x");
e = registerRaise(e.entry, 2 * FLAP_WINDOW_MS + 2 * MIN, 3, "x");
assert.equal(e.verdict, "pass");
assert.equal(e.entry.raises.length, 1, "only the latest is within the window");
let off = registerRaise(undefined, 0, 0, "x");
for (let i = 1; i < 10; i++) off = registerRaise(off.entry, i * MIN, 0, "x");
assert.equal(off.verdict, "pass");
});
test("a recovery is held only while the alert is flapping, and it counts as a moment for the quiet to be measured from", () => {
assert.equal(registerRecovery(undefined, 5).hold, false);
assert.equal(registerRecovery(entry({ raises: [1], flapping: false }), 5).hold, false);
const held = registerRecovery(entry({ flapping: true, lastEvent: 1 }), 50);
assert.deepEqual([held.hold, held.entry?.lastEvent], [true, 50]);
});
test("it has settled after an hour with nothing", () => {
const e = entry({ flapping: true, lastEvent: 0 });
assert.equal(hasSettled(e, SETTLE_MS - 1), false);
assert.equal(hasSettled(e, SETTLE_MS), true);
assert.equal(hasSettled(entry({ flapping: false, lastEvent: 0 }), 10 * SETTLE_MS), false);
});
test("an alert's id says which condition it is about", () => {
assert.equal(keyOfAlertId("disk:disk:server:3:/", "disk"), "disk:server:3:/");
assert.equal(keyOfAlertId("automation:heartbeat:7", "automation"), "heartbeat:7");
});
});
describe("what is sent, what is held, and how it settles", () => {
let hook: Server;
const messages: string[] = [];
const titles = () => messages.map((m) => m.split("\n")[0]);
before(async () => {
hook = createHttpServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c) => chunks.push(c));
req.on("end", () => {
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
res.end("{}");
});
});
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
});
after(() => hook.close());
const issue = { key: "heartbeat:1", message: "Nightly backup hasn't reported", source: "heartbeat", severity: "warning" };
test("raised and cleared again and again: two pairs go out as they are, the third raise is one 'flapping' message, and then it is quiet", async () => {
messages.length = 0;
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
assert.equal(messages.length, 4, "the first two times are as they always were");
await notifyHealthIssues([issue]);
assert.equal(messages.length, 5);
assert.match(messages[4], /^\*\*Homelab Manager — Flapping alert/, "titles go out in bold");
assert.match(messages[4], /↻ Nightly backup hasn't reported — raised 3 times within an hour/);
assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]);
await notifyHealthRecovered([issue]);
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
assert.equal(messages.length, 5, "nothing more while it flaps");
const audit = await db.select().from(auditLog).where(eq(auditLog.action, "flapping_started"));
assert.equal(audit.length, 1);
assert.equal(JSON.parse(audit[0].detail!).key, "heartbeat:1");
});
test("another alert is not affected by it, and one without a key is never held", async () => {
messages.length = 0;
await notifyHealthIssues([{ key: "disk:server:3:/", message: "nas: / is 96% full", severity: "warning" }, { message: "something with no key" }]);
assert.equal(messages.length, 1);
assert.match(messages[0], /nas: \/ is 96% full/);
assert.match(messages[0], /something with no key/);
});
test("the Alerts page marks it as flapping while it is, and not after", async () => {
const [h] = await db.insert(heartbeats).values({ name: "Nightly backup", token: "hb_x".padEnd(24, "x"), periodMinutes: 60, graceMinutes: 5, createdAt: "2026-01-01 00:00:00" }).returning();
// the real heartbeat's key is heartbeat:<id>: make the held alert that one
await setInternalFlag("flapState", JSON.stringify({ [`heartbeat:${h.id}`]: { raises: [Date.now()], flapping: true, since: Date.now(), lastEvent: Date.now(), message: "m" } }));
invalidateAlertsCache();
const alert = (await getAlerts(true)).alerts.find((a) => a.id === `automation:heartbeat:${h.id}`)!;
assert.ok(alert, "the heartbeat is late, so it is an alert");
assert.equal(alert.flapping, true);
await setInternalFlag("flapState", "{}");
invalidateAlertsCache();
assert.equal((await getAlerts(true)).alerts.find((a) => a.id === `automation:heartbeat:${h.id}`)!.flapping, undefined);
await db.delete(heartbeats);
invalidateAlertsCache(); // the heartbeat that was an alert a moment ago is gone
});
test("an hour of quiet settles it: one message says what state it is in; after that it is announced as usual", async () => {
await setInternalFlag("flapState", "{}");
messages.length = 0;
for (let i = 0; i < 3; i++) {
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
}
assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]);
const sent = messages.length;
assert.equal(await settleFlappingAlerts(new Date(Date.now() + 30 * MIN)), 0, "half an hour is not enough");
assert.equal(await settleFlappingAlerts(new Date(Date.now() + SETTLE_MS + MIN)), 1);
assert.equal(messages.length, sent + 1);
assert.match(messages[messages.length - 1], /Alert settled[\s\S]*✓ cleared — Nightly backup hasn't reported/, "nothing says it is still raised, so it says it cleared");
assert.deepEqual([...(await flappingKeys())], []);
assert.equal((await db.select().from(auditLog).where(eq(auditLog.action, "flapping_ended"))).length, 1);
const before = messages.length;
await notifyHealthIssues([issue]);
assert.equal(messages.length, before + 1);
assert.match(messages[messages.length - 1], /Health Alert/);
});
test("turned off, every raise and every recovery is sent", async () => {
await setInternalFlag("flapState", "{}");
await updateSettings({ healthChecks: { flapRaises: 0 } });
messages.length = 0;
for (let i = 0; i < 5; i++) {
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
}
assert.equal(messages.length, 10);
assert.equal(titles().some((t) => t.includes("Flapping")), false);
await updateSettings({ healthChecks: { flapRaises: 3 } });
});
test("with the notification switched off, it is still tracked, so switching it on does not start from nothing", async () => {
await setInternalFlag("flapState", "{}");
await updateSettings({ notifications: { healthAlerts: false } });
messages.length = 0;
for (let i = 0; i < 3; i++) await notifyHealthIssues([issue]);
assert.equal(messages.length, 0);
assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]);
await updateSettings({ notifications: { healthAlerts: true } });
await setInternalFlag("flapState", "{}");
});
});
describe("the noise report", () => {
let app: Server;
let base = "";
const incident = (alertKey: string, extra: Partial<typeof alertIncidents.$inferInsert> = {}) => ({ alertKey, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: alertKey, openedAt: "2026-10-10T00:00:00.000Z", lastSeenAt: "2026-10-10T00:00:00.000Z", ...extra });
const at = (day: number, hour: number, min = 0) => new Date(Date.UTC(2026, 9, day, hour, min)).toISOString();
const NOW = new Date("2026-10-19T12:00:00Z");
before(async () => {
await db.insert(users).values({ oidcSub: "viewer", name: "Vic", role: "viewer" });
await db.delete(alertIncidents);
await db.insert(alertIncidents).values([
// a disk that came and went five times, three of them within forty minutes
...[[10, 1, 0, 5], [10, 1, 15, 20], [10, 1, 35, 40], [12, 8, 0, 30], [15, 8, 0, 10]].map(([d, h, m, len]) => incident("disk:disk:server:3:/", { message: "nas: / is 96% full", targetName: "nas", openedAt: at(d, h, m), resolvedAt: new Date(Date.parse(at(d, h, m)) + len * MIN).toISOString(), link: "/servers/3" })),
incident("expiry:secret:1", { category: "expiry", message: "Secret expires", openedAt: at(11, 8) }),
incident("offline:offline:server:2", { category: "offline", message: "elsa stopped reporting", openedAt: at(5, 8), resolvedAt: at(5, 9) }),
incident("offline:offline:server:2", { category: "offline", message: "elsa stopped reporting", openedAt: at(18, 8) }),
// before the period
incident("disk:disk:server:3:/", { message: "old", openedAt: "2026-05-01T00:00:00.000Z", resolvedAt: "2026-05-01T01:00:00.000Z" }),
]);
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/alerts", alertsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/alerts`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the most times within any hour, however the times are spread", () => {
const h = 3600_000;
assert.equal(mostWithin([], h), 0);
assert.equal(mostWithin([0, 10 * MIN, 20 * MIN, 5 * h], h), 3);
assert.equal(mostWithin([0, h, 2 * h], h), 1, "exactly an hour apart is not within an hour");
assert.equal(mostWithin([5 * h, 0, 1 * MIN], h), 2, "in any order");
});
test("what fired most, how long it lasted, which flapped; one that fired once is not noise; an incident before the period is not counted", async () => {
const r = await buildNoiseReport(30, NOW);
assert.deepEqual([r.days, r.incidents, r.distinctAlerts, r.repeating, r.flapped], [30, 8, 3, 2, 1]);
const disk = r.rows[0];
assert.deepEqual([disk.alertId, disk.fired, disk.totalMinutes, disk.averageMinutes, disk.longestMinutes, disk.mostInAnHour, disk.flapped, disk.openNow], ["disk:disk:server:3:/", 5, 105, 21, 40, 3, true, false]);
assert.equal(disk.message, "nas: / is 96% full", "what it said last");
assert.equal(disk.where, "nas");
assert.match(disk.hint, /Raise the limit for this disk/);
const elsa = r.rows[1];
assert.deepEqual([elsa.fired, elsa.openNow, elsa.flapped], [2, true, false]);
assert.match(elsa.hint, /Server offline after/);
assert.equal(elsa.longestMinutes > 60, true, "the open one has been going since it began");
assert.equal(r.rows.some((x) => x.alertId === "expiry:secret:1"), false);
assert.deepEqual(r.byCategory.map((c) => [c.category, c.incidents, c.repeating]), [["disk", 5, 1], ["offline", 2, 1], ["expiry", 1, 0]]);
assert.equal((await buildNoiseReport(180, NOW)).incidents, 9, "a longer period reaches the old one");
assert.equal((await buildNoiseReport(7, NOW)).incidents, 2, "a shorter one leaves most out");
});
test("an alert that is flapping now says so", async () => {
await setInternalFlag("flapState", JSON.stringify({ "disk:server:3:/": { raises: [Date.now()], flapping: true, since: Date.now(), lastEvent: Date.now(), message: "m" } }));
const r = await buildNoiseReport(30, NOW);
assert.equal(r.rows[0].flappingNow, true);
assert.equal(r.rows[1].flappingNow, false);
await setInternalFlag("flapState", "{}");
});
test("advice for what the alert is about", () => {
assert.match(hintFor("monitoring", "monitoring:new-port:server:1:tcp:8080"), /Ports card/);
assert.match(hintFor("automation", "automation:heartbeat:1"), /grace/);
assert.match(hintFor("monitoring", "monitoring:endpoint:2"), /failures in a row/);
assert.match(hintFor("services", "services:failed-units:server:1"), /Services to watch/);
assert.match(hintFor("tickets", "tickets:x"), /Acknowledge it/);
});
test("everyone signed in can read it; the period is one of four, or thirty", async () => {
const body = await (await fetch(`${base}/noise?days=90`)).json();
assert.equal(body.days, 90);
assert.equal((await (await fetch(`${base}/noise?days=banana`)).json()).days, 30);
assert.equal((await (await fetch(`${base}/noise?days=45`)).json()).days, 30, "not an arbitrary number");
assert.ok(Array.isArray(body.rows));
});
});