Flapping alerts, a noise report and bulk actions with undo
Flapping: an alert raised three times within an hour is announced once as flapping, its further raise/clear messages are held until it has been calm for an hour, and one message then says whether it is still active. Event webhooks still hear everything. The state survives restarts. Alerts > Noise lists the alerts that keep coming back (7-180 days): how often, how long, most in an hour, and a hint on what helps. Bulk actions on Alerts (acknowledge, remove acknowledgement, silence), Servers (tag, silence, runs on) and Inventory (location and dates, print labels). Each reports what it left alone and why, and has an Undo on the toast. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
be6c5552bd
commit
73a06ebe63
35 files changed
+2546
-17
No files matched your search
@@ -0,0 +1,346 @@
|
||||
/**
|
||||
* Bulk actions: the same thing done to several alerts, servers or hardware entries, who may do which, what is skipped and why, and that the body each one hands back
|
||||
* puts everything exactly as it was.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-bulk-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, heartbeats, assets, alertAcks, maintenanceWindows, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { serversBulkRouter, alertsBulkRouter, inventoryBulkRouter } = await import("../src/routes/bulk.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { listActiveWindows } = await import("../src/services/maintenance.js");
|
||||
const { getChangeLog } = await import("../src/services/changeLog.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
|
||||
const send = async (path: string, body: unknown) => {
|
||||
const res = await fetch(`${base}${path}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
|
||||
return { status: res.status, body: (await res.json().catch(() => null)) as any };
|
||||
};
|
||||
const tagsOf = async (id: number) => JSON.parse((await db.select().from(servers).where(eq(servers.id, id)))[0].tags ?? "[]") as string[];
|
||||
|
||||
const ids: Record<string, number> = {};
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/servers/bulk", serversBulkRouter);
|
||||
a.use("/api/alerts/bulk", alertsBulkRouter);
|
||||
a.use("/api/inventory/bulk", inventoryBulkRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
for (const [name, tags] of [["web", ["lab"]], ["api", ["lab", "prod"]], ["db", null], ["host", null]] as const) {
|
||||
ids[name] = (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, tags: tags ? JSON.stringify(tags) : null, lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
}
|
||||
// a guest that runs on host: what depends on it
|
||||
ids.guest = (await db.insert(servers).values({ name: "guest", apiTokenHash: "g", apiTokenPrefix: "g", lastSeenAt: new Date().toISOString(), parentServerId: ids.host }).returning({ id: servers.id }))[0].id;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
describe("servers: tags", () => {
|
||||
test("a tag is added to those that lack it; those that have it and those that have gone are listed with the reason; undo puts the tags back exactly", async () => {
|
||||
const r = await send("/servers/bulk", { action: "add_tag", ids: [ids.web, ids.api, ids.db, 99999], tag: "Nightly" });
|
||||
assert.equal(r.status, 200);
|
||||
assert.equal(r.body.changed, 3);
|
||||
assert.deepEqual(await tagsOf(ids.web), ["lab", "nightly"], "written the way tags are");
|
||||
assert.deepEqual(await tagsOf(ids.db), ["nightly"]);
|
||||
assert.deepEqual(r.body.undo.items.map((i: { id: number; tags: string[] }) => [i.id, i.tags]).sort(), [[ids.web, ["lab"]], [ids.api, ["lab", "prod"]], [ids.db, []]].sort());
|
||||
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it no longer exists"]);
|
||||
|
||||
const again = await send("/servers/bulk", { action: "add_tag", ids: [ids.web, ids.db], tag: "nightly" });
|
||||
assert.equal(again.body.changed, 0);
|
||||
assert.equal(again.body.undo, null, "nothing changed, so nothing to undo");
|
||||
assert.deepEqual(again.body.skipped.map((s: { reason: string }) => s.reason), ["it already has that tag", "it already has that tag"]);
|
||||
|
||||
const undone = await send("/servers/bulk", r.body.undo);
|
||||
assert.equal(undone.body.changed, 3);
|
||||
assert.deepEqual([await tagsOf(ids.web), await tagsOf(ids.api), await tagsOf(ids.db)], [["lab"], ["lab", "prod"], []]);
|
||||
assert.equal((await db.select().from(servers).where(eq(servers.id, ids.db)))[0].tags, null, "no tags is none, not an empty list");
|
||||
});
|
||||
|
||||
test("a tag is taken off those that have it", async () => {
|
||||
const r = await send("/servers/bulk", { action: "remove_tag", ids: [ids.web, ids.api, ids.db], tag: "lab" });
|
||||
assert.equal(r.body.changed, 2);
|
||||
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it doesn't have that tag"]);
|
||||
assert.deepEqual([await tagsOf(ids.web), await tagsOf(ids.api)], [[], ["prod"]]);
|
||||
await send("/servers/bulk", r.body.undo);
|
||||
assert.deepEqual([await tagsOf(ids.web), await tagsOf(ids.api)], [["lab"], ["lab", "prod"]]);
|
||||
});
|
||||
|
||||
test("a wrong tag is refused with the reason; a server with no room for another is skipped; the rest are done", async () => {
|
||||
assert.equal((await send("/servers/bulk", { action: "add_tag", ids: [ids.web], tag: "bad tag!" })).status, 400);
|
||||
const full = Array.from({ length: 12 }, (_, i) => `t${String(i).padStart(2, "0")}`);
|
||||
await db.update(servers).set({ tags: JSON.stringify(full) }).where(eq(servers.id, ids.host));
|
||||
const r = await send("/servers/bulk", { action: "add_tag", ids: [ids.host, ids.db], tag: "extra" });
|
||||
assert.equal(r.body.changed, 1);
|
||||
assert.match(r.body.skipped[0].reason, /at most 12 tags/);
|
||||
await send("/servers/bulk", r.body.undo);
|
||||
await db.update(servers).set({ tags: null }).where(eq(servers.id, ids.host));
|
||||
});
|
||||
});
|
||||
|
||||
describe("servers: maintenance", () => {
|
||||
test("each is put under maintenance, one already under it is extended; undo ends the new and puts the old back as it was", async () => {
|
||||
const T = new Date();
|
||||
await db.delete(maintenanceWindows);
|
||||
const before = (await db.insert(maintenanceWindows).values({ targetType: "server", targetId: ids.db, reason: "earlier", startedAt: new Date(T.getTime() - 60_000).toISOString(), endsAt: new Date(T.getTime() + 10 * 60_000).toISOString() }).returning())[0];
|
||||
const r = await send("/servers/bulk", { action: "silence", ids: [ids.web, ids.db], minutes: 90, reason: "Patching" });
|
||||
assert.equal(r.status, 200);
|
||||
assert.equal(r.body.changed, 2);
|
||||
assert.deepEqual(r.body.silenced.map((s: { name: string }) => s.name).sort(), ["db", "web"]);
|
||||
const during = await listActiveWindows();
|
||||
assert.equal(during.some((w) => w.targetId === ids.web), true);
|
||||
assert.ok(Date.parse(during.find((w) => w.targetId === ids.db)!.endsAt) > T.getTime() + 60 * 60_000, "extended to ninety minutes");
|
||||
assert.equal(r.body.undo.created.length, 1);
|
||||
assert.deepEqual(r.body.undo.restored.map((x: { id: number; endsAt: string }) => [x.id, x.endsAt]), [[before.id, before.endsAt]]);
|
||||
|
||||
const undone = await send("/servers/bulk", r.body.undo);
|
||||
assert.equal(undone.body.changed, 2);
|
||||
const after = await listActiveWindows();
|
||||
assert.equal(after.some((w) => w.targetId === ids.web), false, "the one it made is over");
|
||||
const old = (await db.select().from(maintenanceWindows).where(eq(maintenanceWindows.id, before.id)))[0];
|
||||
assert.deepEqual([old.endsAt, old.reason], [before.endsAt, "earlier"], "the one that was there is as it was");
|
||||
});
|
||||
|
||||
test("what depends on a server can be silenced with it", async () => {
|
||||
await db.delete(maintenanceWindows);
|
||||
const r = await send("/servers/bulk", { action: "silence", ids: [ids.host], minutes: 30, withDependents: true });
|
||||
assert.deepEqual(r.body.silenced.map((s: { name: string }) => s.name).sort(), ["guest", "host"]);
|
||||
assert.match((await listActiveWindows()).find((w) => w.targetId === ids.guest)!.reason ?? "", /^Depends on host/);
|
||||
await send("/servers/bulk", r.body.undo);
|
||||
assert.equal((await listActiveWindows()).length, 0);
|
||||
});
|
||||
|
||||
test("how long must be sensible", async () => {
|
||||
assert.equal((await send("/servers/bulk", { action: "silence", ids: [ids.web], minutes: 1 })).status, 400);
|
||||
assert.equal((await send("/servers/bulk", { action: "silence", ids: [ids.web], minutes: 99999 })).status, 400);
|
||||
});
|
||||
});
|
||||
|
||||
describe("servers: which server another runs on", () => {
|
||||
test("an administrator sets it for several; a loop is skipped; undo restores each exactly; an operator may not", async () => {
|
||||
who = "operator";
|
||||
assert.equal((await send("/servers/bulk", { action: "set_parent", ids: [ids.web], parentServerId: ids.host })).status, 403);
|
||||
who = "admin";
|
||||
const r = await send("/servers/bulk", { action: "set_parent", ids: [ids.web, ids.api, ids.host], parentServerId: ids.host });
|
||||
assert.equal(r.status, 200);
|
||||
assert.equal(r.body.changed, 2);
|
||||
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["a server can't run on itself"]);
|
||||
const parent = async (id: number) => (await db.select().from(servers).where(eq(servers.id, id)))[0].parentServerId;
|
||||
assert.deepEqual([await parent(ids.web), await parent(ids.api)], [ids.host, ids.host]);
|
||||
|
||||
// guest already runs on host: making host run on guest would close a loop
|
||||
const loop = await send("/servers/bulk", { action: "set_parent", ids: [ids.host], parentServerId: ids.guest });
|
||||
assert.equal(loop.body.changed, 0);
|
||||
assert.match(loop.body.skipped[0].reason, /each would be waiting for the other/);
|
||||
|
||||
assert.equal((await send("/servers/bulk", { action: "set_parent", ids: [ids.web], parentServerId: 99999 })).status, 400);
|
||||
await send("/servers/bulk", r.body.undo);
|
||||
assert.deepEqual([await parent(ids.web), await parent(ids.api)], [null, null]);
|
||||
const auto = await send("/servers/bulk", { action: "set_parent", ids: [ids.web], parentServerId: null });
|
||||
assert.equal(auto.body.changed, 1, "null goes back to working it out");
|
||||
who = "operator";
|
||||
});
|
||||
|
||||
test("a viewer can do none of it", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await send("/servers/bulk", { action: "add_tag", ids: [ids.web], tag: "x" })).status, 403);
|
||||
assert.equal((await send("/servers/bulk", { action: "silence", ids: [ids.web], minutes: 30 })).status, 403);
|
||||
who = "operator";
|
||||
});
|
||||
|
||||
test("too many, none, or something that isn't an action are refused", async () => {
|
||||
assert.equal((await send("/servers/bulk", { action: "add_tag", ids: [], tag: "x" })).status, 400);
|
||||
assert.equal((await send("/servers/bulk", { action: "add_tag", ids: Array.from({ length: 201 }, (_, i) => i + 1), tag: "x" })).status, 400);
|
||||
assert.equal((await send("/servers/bulk", { action: "delete_everything", ids: [1] })).status, 400);
|
||||
});
|
||||
});
|
||||
|
||||
describe("alerts", () => {
|
||||
const alertIds: string[] = [];
|
||||
|
||||
before(async () => {
|
||||
await db.delete(maintenanceWindows);
|
||||
// three late heartbeats: two on servers (so each has something to silence, and both on the same one), one on nothing
|
||||
const old = "2026-01-01 00:00:00";
|
||||
const mk = async (name: string, serverId: number | null) => (await db.insert(heartbeats).values({ name, token: `hb_${name}`.padEnd(24, "x"), periodMinutes: 60, graceMinutes: 5, serverId, createdAt: old }).returning())[0];
|
||||
const h1 = await mk("job1", ids.web);
|
||||
const h2 = await mk("job2", ids.web);
|
||||
const h3 = await mk("job3", null);
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
for (const h of [h1, h2, h3]) alertIds.push(report.alerts.find((a) => a.message.startsWith(h.name))!.id);
|
||||
});
|
||||
|
||||
test("several are acknowledged the same way; one that has gone is skipped; undo takes them off again", async () => {
|
||||
const r = await send("/alerts/bulk", { action: "acknowledge", alertIds: [...alertIds, "automation:heartbeat:99999"], note: "Known, fixing tonight", hours: 12 });
|
||||
assert.equal(r.status, 200);
|
||||
assert.equal(r.body.changed, 3);
|
||||
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it isn't there any more"]);
|
||||
const rows = await db.select().from(alertAcks);
|
||||
assert.equal(rows.length, 3);
|
||||
assert.ok(rows.every((x) => x.note === "Known, fixing tonight" && x.ackedBy === "Olga" && x.until !== null));
|
||||
assert.ok(r.body.undo.items.every((i: { previous: unknown }) => i.previous === null));
|
||||
|
||||
await send("/alerts/bulk", r.body.undo);
|
||||
assert.equal((await db.select().from(alertAcks)).length, 0);
|
||||
});
|
||||
|
||||
test("an earlier acknowledgement is put back exactly — its note, who it was handed to and when it ends", async () => {
|
||||
await send("/alerts/bulk", { action: "acknowledge", alertIds: [alertIds[0]], note: "first", hours: 5 });
|
||||
const [first] = await db.select().from(alertAcks);
|
||||
const over = await send("/alerts/bulk", { action: "acknowledge", alertIds: [alertIds[0], alertIds[1]], note: "second" });
|
||||
assert.equal(over.body.changed, 2);
|
||||
assert.equal((await db.select().from(alertAcks).where(eq(alertAcks.alertKey, alertIds[0])))[0].note, "second");
|
||||
await send("/alerts/bulk", over.body.undo);
|
||||
const rows = await db.select().from(alertAcks);
|
||||
assert.equal(rows.length, 1, "the second alert had none before");
|
||||
assert.deepEqual([rows[0].note, rows[0].ackedAt, rows[0].until, rows[0].ackedBy], [first.note, first.ackedAt, first.until, first.ackedBy]);
|
||||
});
|
||||
|
||||
test("acknowledgements are taken off, those with none are skipped, and undo puts them back", async () => {
|
||||
await db.delete(alertAcks);
|
||||
await send("/alerts/bulk", { action: "acknowledge", alertIds: [alertIds[0], alertIds[1]], note: "mine" });
|
||||
const r = await send("/alerts/bulk", { action: "unacknowledge", alertIds });
|
||||
assert.equal(r.body.changed, 2);
|
||||
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["nobody had acknowledged it"]);
|
||||
assert.equal((await db.select().from(alertAcks)).length, 0);
|
||||
await send("/alerts/bulk", r.body.undo);
|
||||
assert.deepEqual((await db.select().from(alertAcks)).map((x) => x.note), ["mine", "mine"]);
|
||||
await db.delete(alertAcks);
|
||||
});
|
||||
|
||||
test("handing them to someone who cannot act is refused, and nothing changes", async () => {
|
||||
const r = await send("/alerts/bulk", { action: "acknowledge", alertIds, assigneeId: (await db.select().from(users).where(eq(users.oidcSub, "viewer")))[0].id });
|
||||
assert.equal(r.status, 400);
|
||||
assert.equal((await db.select().from(alertAcks)).length, 0);
|
||||
});
|
||||
|
||||
test("silencing puts each distinct target under maintenance once; an alert with nothing to silence is skipped; undo ends it", async () => {
|
||||
await db.delete(maintenanceWindows);
|
||||
const r = await send("/alerts/bulk", { action: "silence", alertIds, minutes: 60, reason: "Weekend" });
|
||||
assert.equal(r.status, 200);
|
||||
assert.equal(r.body.changed, 1, "two alerts, one server");
|
||||
assert.equal(r.body.silenced[0].name, "web");
|
||||
assert.equal(r.body.skipped.length, 1);
|
||||
assert.match(r.body.skipped[0].reason, /nothing to put under maintenance/);
|
||||
assert.equal((await listActiveWindows()).length, 1);
|
||||
await send("/alerts/bulk", r.body.undo);
|
||||
assert.equal((await listActiveWindows()).length, 0);
|
||||
});
|
||||
|
||||
test("an operator is needed; a viewer is not enough", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await send("/alerts/bulk", { action: "acknowledge", alertIds })).status, 403);
|
||||
who = "operator";
|
||||
});
|
||||
});
|
||||
|
||||
describe("hardware", () => {
|
||||
const hw: Record<string, number> = {};
|
||||
const row = async (id: number) => (await db.select().from(assets).where(eq(assets.id, id)))[0];
|
||||
before(async () => {
|
||||
const now = new Date().toISOString();
|
||||
for (const [name, location, warrantyUntil] of [["NAS", "Rack A", "2027-01-01"], ["Switch", null, null], ["UPS", "Hall", "2026-12-01"]] as const) {
|
||||
hw[name] = (await db.insert(assets).values({ name, kind: "other", location, warrantyUntil, createdAt: now, updatedAt: now } as never).returning({ id: assets.id }))[0].id;
|
||||
}
|
||||
});
|
||||
|
||||
test("a location is set on several at once, a blank clears it, and undo puts each one's own back", async () => {
|
||||
const r = await send("/inventory/bulk", { action: "change", ids: [hw.NAS, hw.Switch, hw.UPS, 99999], location: "Closet" });
|
||||
assert.equal(r.status, 200);
|
||||
assert.equal(r.body.changed, 3);
|
||||
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it no longer exists"]);
|
||||
assert.deepEqual([(await row(hw.NAS)).location, (await row(hw.Switch)).location], ["Closet", "Closet"]);
|
||||
await send("/inventory/bulk", r.body.undo);
|
||||
assert.deepEqual([(await row(hw.NAS)).location, (await row(hw.Switch)).location, (await row(hw.UPS)).location], ["Rack A", null, "Hall"]);
|
||||
const cleared = await send("/inventory/bulk", { action: "change", ids: [hw.NAS], location: " " });
|
||||
assert.equal((await row(hw.NAS)).location, null);
|
||||
await send("/inventory/bulk", cleared.body.undo);
|
||||
assert.equal((await row(hw.NAS)).location, "Rack A");
|
||||
});
|
||||
|
||||
test("the end of support and the warranty are set together or one at a time; whatever is not named is left alone; undo restores", async () => {
|
||||
const r = await send("/inventory/bulk", { action: "change", ids: [hw.NAS, hw.Switch], endOfSupport: "2029-06-30" });
|
||||
assert.equal(r.body.changed, 2);
|
||||
const nas = await row(hw.NAS);
|
||||
assert.deepEqual([nas.endOfSupport, nas.warrantyUntil, nas.location], ["2029-06-30", "2027-01-01", "Rack A"]);
|
||||
await send("/inventory/bulk", { action: "change", ids: [hw.NAS], warrantyUntil: null });
|
||||
assert.equal((await row(hw.NAS)).warrantyUntil, null);
|
||||
await send("/inventory/bulk", r.body.undo);
|
||||
assert.equal((await row(hw.NAS)).endOfSupport, null);
|
||||
assert.equal((await row(hw.Switch)).endOfSupport, null);
|
||||
});
|
||||
|
||||
test("a date that is not a date, or nothing to change, is refused and nothing changes", async () => {
|
||||
assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS], endOfSupport: "2029-02-30" })).status, 400);
|
||||
assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS], warrantyUntil: "soon" })).status, 400);
|
||||
assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS] })).status, 400);
|
||||
assert.equal((await row(hw.NAS)).location, "Rack A");
|
||||
});
|
||||
|
||||
test("an operator is needed", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS], location: "x" })).status, 403);
|
||||
who = "operator";
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is written down", () => {
|
||||
test("each bulk action is one audit entry saying what and how many, and the change log reads them as sentences", async () => {
|
||||
const entries = await db.select().from(auditLog);
|
||||
const actions = new Set(entries.map((e) => `${e.category}/${e.action}`));
|
||||
for (const a of ["server/bulk_tag", "maintenance/bulk_start", "maintenance/bulk_undo", "server/bulk_runs_on", "alert/bulk_ack", "alert/bulk_unack", "alert/bulk_undo", "inventory/bulk_change"]) assert.ok(actions.has(a), a);
|
||||
const tag = entries.find((e) => e.action === "bulk_tag" && JSON.parse(e.detail!).change === "add_tag")!;
|
||||
assert.equal(JSON.parse(tag.detail!).tag, "Nightly");
|
||||
assert.equal(JSON.parse(tag.detail!).changed, 3);
|
||||
|
||||
const log = await getChangeLog({ days: 1, limit: 500 });
|
||||
const titles = log.entries.map((e) => e.title);
|
||||
assert.ok(titles.some((t) => /^Tag added to 3 servers: Nightly$/.test(t)), "tag");
|
||||
assert.ok(titles.some((t) => /^Maintenance started on several at once/.test(t)), "maintenance");
|
||||
assert.ok(titles.some((t) => /alerts? acknowledged at once$/.test(t)), "ack");
|
||||
assert.ok(titles.some((t) => /pieces? of hardware changed at once$/.test(t)), "hardware");
|
||||
assert.ok(titles.some((t) => /was undone$/.test(t)), "undo");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,287 @@
|
||||
/**
|
||||
* Flapping and the noise report: when an alert counts as flapping, what is sent and what is held, how it settles, the flag on the Alerts page, and the report of
|
||||
* which alerts fire most.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-flapping-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, heartbeats, alertIncidents, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { registerRaise, registerRecovery, hasSettled, flappingKeys, FLAP_WINDOW_MS, SETTLE_MS, keyOfAlertId } = await import("../src/services/flapGuard.js");
|
||||
const { settleFlappingAlerts } = await import("../src/services/flapMonitor.js");
|
||||
const { notifyHealthIssues, notifyHealthRecovered } = await import("../src/services/notify.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { buildNoiseReport, mostWithin, hintFor } = await import("../src/services/alertNoise.js");
|
||||
const { alertsRouter } = await import("../src/routes/alerts.js");
|
||||
const { updateSettings, setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const entry = (extra: Partial<Parameters<typeof hasSettled>[0]> = {}) => ({ raises: [], flapping: false, since: null, lastEvent: 0, message: "m", ...extra });
|
||||
|
||||
describe("when an alert counts as flapping", () => {
|
||||
test("raised the threshold number of times within an hour: the third is the one that is called flapping; after that, raises are held", () => {
|
||||
let e = registerRaise(undefined, 0, 3, "disk");
|
||||
assert.equal(e.verdict, "pass");
|
||||
e = registerRaise(e.entry, 10 * MIN, 3, "disk");
|
||||
assert.equal(e.verdict, "pass");
|
||||
e = registerRaise(e.entry, 20 * MIN, 3, "disk");
|
||||
assert.equal(e.verdict, "started");
|
||||
assert.deepEqual([e.entry.flapping, e.entry.since, e.entry.raises.length], [true, 20 * MIN, 3]);
|
||||
e = registerRaise(e.entry, 25 * MIN, 3, "disk");
|
||||
assert.equal(e.verdict, "held");
|
||||
assert.equal(e.entry.flapping, true);
|
||||
});
|
||||
|
||||
test("raises more than an hour apart never add up; a threshold of 0 turns it off", () => {
|
||||
let e = registerRaise(undefined, 0, 3, "x");
|
||||
e = registerRaise(e.entry, FLAP_WINDOW_MS + MIN, 3, "x");
|
||||
e = registerRaise(e.entry, 2 * FLAP_WINDOW_MS + 2 * MIN, 3, "x");
|
||||
assert.equal(e.verdict, "pass");
|
||||
assert.equal(e.entry.raises.length, 1, "only the latest is within the window");
|
||||
let off = registerRaise(undefined, 0, 0, "x");
|
||||
for (let i = 1; i < 10; i++) off = registerRaise(off.entry, i * MIN, 0, "x");
|
||||
assert.equal(off.verdict, "pass");
|
||||
});
|
||||
|
||||
test("a recovery is held only while the alert is flapping, and it counts as a moment for the quiet to be measured from", () => {
|
||||
assert.equal(registerRecovery(undefined, 5).hold, false);
|
||||
assert.equal(registerRecovery(entry({ raises: [1], flapping: false }), 5).hold, false);
|
||||
const held = registerRecovery(entry({ flapping: true, lastEvent: 1 }), 50);
|
||||
assert.deepEqual([held.hold, held.entry?.lastEvent], [true, 50]);
|
||||
});
|
||||
|
||||
test("it has settled after an hour with nothing", () => {
|
||||
const e = entry({ flapping: true, lastEvent: 0 });
|
||||
assert.equal(hasSettled(e, SETTLE_MS - 1), false);
|
||||
assert.equal(hasSettled(e, SETTLE_MS), true);
|
||||
assert.equal(hasSettled(entry({ flapping: false, lastEvent: 0 }), 10 * SETTLE_MS), false);
|
||||
});
|
||||
|
||||
test("an alert's id says which condition it is about", () => {
|
||||
assert.equal(keyOfAlertId("disk:disk:server:3:/", "disk"), "disk:server:3:/");
|
||||
assert.equal(keyOfAlertId("automation:heartbeat:7", "automation"), "heartbeat:7");
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is sent, what is held, and how it settles", () => {
|
||||
let hook: Server;
|
||||
const messages: string[] = [];
|
||||
const titles = () => messages.map((m) => m.split("\n")[0]);
|
||||
|
||||
before(async () => {
|
||||
hook = createHttpServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
|
||||
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
|
||||
});
|
||||
|
||||
after(() => hook.close());
|
||||
|
||||
const issue = { key: "heartbeat:1", message: "Nightly backup hasn't reported", source: "heartbeat", severity: "warning" };
|
||||
|
||||
test("raised and cleared again and again: two pairs go out as they are, the third raise is one 'flapping' message, and then it is quiet", async () => {
|
||||
messages.length = 0;
|
||||
await notifyHealthIssues([issue]);
|
||||
await notifyHealthRecovered([issue]);
|
||||
await notifyHealthIssues([issue]);
|
||||
await notifyHealthRecovered([issue]);
|
||||
assert.equal(messages.length, 4, "the first two times are as they always were");
|
||||
await notifyHealthIssues([issue]);
|
||||
assert.equal(messages.length, 5);
|
||||
assert.match(messages[4], /^\*\*Homelab Manager — Flapping alert/, "titles go out in bold");
|
||||
assert.match(messages[4], /↻ Nightly backup hasn't reported — raised 3 times within an hour/);
|
||||
assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]);
|
||||
|
||||
await notifyHealthRecovered([issue]);
|
||||
await notifyHealthIssues([issue]);
|
||||
await notifyHealthRecovered([issue]);
|
||||
assert.equal(messages.length, 5, "nothing more while it flaps");
|
||||
const audit = await db.select().from(auditLog).where(eq(auditLog.action, "flapping_started"));
|
||||
assert.equal(audit.length, 1);
|
||||
assert.equal(JSON.parse(audit[0].detail!).key, "heartbeat:1");
|
||||
});
|
||||
|
||||
test("another alert is not affected by it, and one without a key is never held", async () => {
|
||||
messages.length = 0;
|
||||
await notifyHealthIssues([{ key: "disk:server:3:/", message: "nas: / is 96% full", severity: "warning" }, { message: "something with no key" }]);
|
||||
assert.equal(messages.length, 1);
|
||||
assert.match(messages[0], /nas: \/ is 96% full/);
|
||||
assert.match(messages[0], /something with no key/);
|
||||
});
|
||||
|
||||
test("the Alerts page marks it as flapping while it is, and not after", async () => {
|
||||
const [h] = await db.insert(heartbeats).values({ name: "Nightly backup", token: "hb_x".padEnd(24, "x"), periodMinutes: 60, graceMinutes: 5, createdAt: "2026-01-01 00:00:00" }).returning();
|
||||
// the real heartbeat's key is heartbeat:<id>: make the held alert that one
|
||||
await setInternalFlag("flapState", JSON.stringify({ [`heartbeat:${h.id}`]: { raises: [Date.now()], flapping: true, since: Date.now(), lastEvent: Date.now(), message: "m" } }));
|
||||
invalidateAlertsCache();
|
||||
const alert = (await getAlerts(true)).alerts.find((a) => a.id === `automation:heartbeat:${h.id}`)!;
|
||||
assert.ok(alert, "the heartbeat is late, so it is an alert");
|
||||
assert.equal(alert.flapping, true);
|
||||
await setInternalFlag("flapState", "{}");
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await getAlerts(true)).alerts.find((a) => a.id === `automation:heartbeat:${h.id}`)!.flapping, undefined);
|
||||
await db.delete(heartbeats);
|
||||
invalidateAlertsCache(); // the heartbeat that was an alert a moment ago is gone
|
||||
});
|
||||
|
||||
test("an hour of quiet settles it: one message says what state it is in; after that it is announced as usual", async () => {
|
||||
await setInternalFlag("flapState", "{}");
|
||||
messages.length = 0;
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await notifyHealthIssues([issue]);
|
||||
await notifyHealthRecovered([issue]);
|
||||
}
|
||||
assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]);
|
||||
const sent = messages.length;
|
||||
assert.equal(await settleFlappingAlerts(new Date(Date.now() + 30 * MIN)), 0, "half an hour is not enough");
|
||||
assert.equal(await settleFlappingAlerts(new Date(Date.now() + SETTLE_MS + MIN)), 1);
|
||||
assert.equal(messages.length, sent + 1);
|
||||
assert.match(messages[messages.length - 1], /Alert settled[\s\S]*✓ cleared — Nightly backup hasn't reported/, "nothing says it is still raised, so it says it cleared");
|
||||
assert.deepEqual([...(await flappingKeys())], []);
|
||||
assert.equal((await db.select().from(auditLog).where(eq(auditLog.action, "flapping_ended"))).length, 1);
|
||||
const before = messages.length;
|
||||
await notifyHealthIssues([issue]);
|
||||
assert.equal(messages.length, before + 1);
|
||||
assert.match(messages[messages.length - 1], /Health Alert/);
|
||||
});
|
||||
|
||||
test("turned off, every raise and every recovery is sent", async () => {
|
||||
await setInternalFlag("flapState", "{}");
|
||||
await updateSettings({ healthChecks: { flapRaises: 0 } });
|
||||
messages.length = 0;
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await notifyHealthIssues([issue]);
|
||||
await notifyHealthRecovered([issue]);
|
||||
}
|
||||
assert.equal(messages.length, 10);
|
||||
assert.equal(titles().some((t) => t.includes("Flapping")), false);
|
||||
await updateSettings({ healthChecks: { flapRaises: 3 } });
|
||||
});
|
||||
|
||||
test("with the notification switched off, it is still tracked, so switching it on does not start from nothing", async () => {
|
||||
await setInternalFlag("flapState", "{}");
|
||||
await updateSettings({ notifications: { healthAlerts: false } });
|
||||
messages.length = 0;
|
||||
for (let i = 0; i < 3; i++) await notifyHealthIssues([issue]);
|
||||
assert.equal(messages.length, 0);
|
||||
assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]);
|
||||
await updateSettings({ notifications: { healthAlerts: true } });
|
||||
await setInternalFlag("flapState", "{}");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the noise report", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const incident = (alertKey: string, extra: Partial<typeof alertIncidents.$inferInsert> = {}) => ({ alertKey, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: alertKey, openedAt: "2026-10-10T00:00:00.000Z", lastSeenAt: "2026-10-10T00:00:00.000Z", ...extra });
|
||||
const at = (day: number, hour: number, min = 0) => new Date(Date.UTC(2026, 9, day, hour, min)).toISOString();
|
||||
const NOW = new Date("2026-10-19T12:00:00Z");
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values({ oidcSub: "viewer", name: "Vic", role: "viewer" });
|
||||
await db.delete(alertIncidents);
|
||||
await db.insert(alertIncidents).values([
|
||||
// a disk that came and went five times, three of them within forty minutes
|
||||
...[[10, 1, 0, 5], [10, 1, 15, 20], [10, 1, 35, 40], [12, 8, 0, 30], [15, 8, 0, 10]].map(([d, h, m, len]) => incident("disk:disk:server:3:/", { message: "nas: / is 96% full", targetName: "nas", openedAt: at(d, h, m), resolvedAt: new Date(Date.parse(at(d, h, m)) + len * MIN).toISOString(), link: "/servers/3" })),
|
||||
incident("expiry:secret:1", { category: "expiry", message: "Secret expires", openedAt: at(11, 8) }),
|
||||
incident("offline:offline:server:2", { category: "offline", message: "elsa stopped reporting", openedAt: at(5, 8), resolvedAt: at(5, 9) }),
|
||||
incident("offline:offline:server:2", { category: "offline", message: "elsa stopped reporting", openedAt: at(18, 8) }),
|
||||
// before the period
|
||||
incident("disk:disk:server:3:/", { message: "old", openedAt: "2026-05-01T00:00:00.000Z", resolvedAt: "2026-05-01T01:00:00.000Z" }),
|
||||
]);
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/alerts", alertsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/alerts`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("the most times within any hour, however the times are spread", () => {
|
||||
const h = 3600_000;
|
||||
assert.equal(mostWithin([], h), 0);
|
||||
assert.equal(mostWithin([0, 10 * MIN, 20 * MIN, 5 * h], h), 3);
|
||||
assert.equal(mostWithin([0, h, 2 * h], h), 1, "exactly an hour apart is not within an hour");
|
||||
assert.equal(mostWithin([5 * h, 0, 1 * MIN], h), 2, "in any order");
|
||||
});
|
||||
|
||||
test("what fired most, how long it lasted, which flapped; one that fired once is not noise; an incident before the period is not counted", async () => {
|
||||
const r = await buildNoiseReport(30, NOW);
|
||||
assert.deepEqual([r.days, r.incidents, r.distinctAlerts, r.repeating, r.flapped], [30, 8, 3, 2, 1]);
|
||||
const disk = r.rows[0];
|
||||
assert.deepEqual([disk.alertId, disk.fired, disk.totalMinutes, disk.averageMinutes, disk.longestMinutes, disk.mostInAnHour, disk.flapped, disk.openNow], ["disk:disk:server:3:/", 5, 105, 21, 40, 3, true, false]);
|
||||
assert.equal(disk.message, "nas: / is 96% full", "what it said last");
|
||||
assert.equal(disk.where, "nas");
|
||||
assert.match(disk.hint, /Raise the limit for this disk/);
|
||||
const elsa = r.rows[1];
|
||||
assert.deepEqual([elsa.fired, elsa.openNow, elsa.flapped], [2, true, false]);
|
||||
assert.match(elsa.hint, /Server offline after/);
|
||||
assert.equal(elsa.longestMinutes > 60, true, "the open one has been going since it began");
|
||||
assert.equal(r.rows.some((x) => x.alertId === "expiry:secret:1"), false);
|
||||
assert.deepEqual(r.byCategory.map((c) => [c.category, c.incidents, c.repeating]), [["disk", 5, 1], ["offline", 2, 1], ["expiry", 1, 0]]);
|
||||
assert.equal((await buildNoiseReport(180, NOW)).incidents, 9, "a longer period reaches the old one");
|
||||
assert.equal((await buildNoiseReport(7, NOW)).incidents, 2, "a shorter one leaves most out");
|
||||
});
|
||||
|
||||
test("an alert that is flapping now says so", async () => {
|
||||
await setInternalFlag("flapState", JSON.stringify({ "disk:server:3:/": { raises: [Date.now()], flapping: true, since: Date.now(), lastEvent: Date.now(), message: "m" } }));
|
||||
const r = await buildNoiseReport(30, NOW);
|
||||
assert.equal(r.rows[0].flappingNow, true);
|
||||
assert.equal(r.rows[1].flappingNow, false);
|
||||
await setInternalFlag("flapState", "{}");
|
||||
});
|
||||
|
||||
test("advice for what the alert is about", () => {
|
||||
assert.match(hintFor("monitoring", "monitoring:new-port:server:1:tcp:8080"), /Ports card/);
|
||||
assert.match(hintFor("automation", "automation:heartbeat:1"), /grace/);
|
||||
assert.match(hintFor("monitoring", "monitoring:endpoint:2"), /failures in a row/);
|
||||
assert.match(hintFor("services", "services:failed-units:server:1"), /Services to ignore/);
|
||||
assert.match(hintFor("tickets", "tickets:x"), /Acknowledge it/);
|
||||
});
|
||||
|
||||
test("everyone signed in can read it; the period is one of four, or thirty", async () => {
|
||||
const body = await (await fetch(`${base}/noise?days=90`)).json();
|
||||
assert.equal(body.days, 90);
|
||||
assert.equal((await (await fetch(`${base}/noise?days=banana`)).json()).days, 30);
|
||||
assert.equal((await (await fetch(`${base}/noise?days=45`)).json()).days, 30, "not an arbitrary number");
|
||||
assert.ok(Array.isArray(body.rows));
|
||||
});
|
||||
});
|
||||
@@ -174,6 +174,8 @@ describe("through the channels", () => {
|
||||
gotify: { enabled: true, url: base.gotify, token: "g" },
|
||||
ntfy: { enabled: true, url: base.ntfy, topic: "lab" },
|
||||
webhook: { enabled: true, url: base.webhook },
|
||||
// These tests raise the same problem again and again as fixtures; they are about routing, not about flapping.
|
||||
healthChecks: { flapRaises: 0 },
|
||||
});
|
||||
proddy = (await db.insert(servers).values({ name: "prod-db", apiTokenHash: "p", apiTokenPrefix: "p", tags: JSON.stringify(["prod"]) }).returning({ id: servers.id }))[0].id;
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user