Flapping alerts, a noise report and bulk actions with undo

Flapping: an alert raised three times within an hour is announced once as
flapping, its further raise/clear messages are held until it has been calm
for an hour, and one message then says whether it is still active. Event
webhooks still hear everything. The state survives restarts.

Alerts > Noise lists the alerts that keep coming back (7-180 days): how often,
how long, most in an hour, and a hint on what helps.

Bulk actions on Alerts (acknowledge, remove acknowledgement, silence), Servers
(tag, silence, runs on) and Inventory (location and dates, print labels). Each
reports what it left alone and why, and has an Undo on the toast.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 23:55:09 +02:00
1 parent be6c5552bd
commit 73a06ebe63
35 files changed
+2546 -17

No files matched your search

+346
View File
@@ -0,0 +1,346 @@
/**
* Bulk actions: the same thing done to several alerts, servers or hardware entries, who may do which, what is skipped and why, and that the body each one hands back
* puts everything exactly as it was.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-bulk-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, heartbeats, assets, alertAcks, maintenanceWindows, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { serversBulkRouter, alertsBulkRouter, inventoryBulkRouter } = await import("../src/routes/bulk.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { listActiveWindows } = await import("../src/services/maintenance.js");
const { getChangeLog } = await import("../src/services/changeLog.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
let app: Server;
let base = "";
let who = "operator";
const send = async (path: string, body: unknown) => {
const res = await fetch(`${base}${path}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
return { status: res.status, body: (await res.json().catch(() => null)) as any };
};
const tagsOf = async (id: number) => JSON.parse((await db.select().from(servers).where(eq(servers.id, id)))[0].tags ?? "[]") as string[];
const ids: Record<string, number> = {};
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/servers/bulk", serversBulkRouter);
a.use("/api/alerts/bulk", alertsBulkRouter);
a.use("/api/inventory/bulk", inventoryBulkRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
for (const [name, tags] of [["web", ["lab"]], ["api", ["lab", "prod"]], ["db", null], ["host", null]] as const) {
ids[name] = (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, tags: tags ? JSON.stringify(tags) : null, lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
}
// a guest that runs on host: what depends on it
ids.guest = (await db.insert(servers).values({ name: "guest", apiTokenHash: "g", apiTokenPrefix: "g", lastSeenAt: new Date().toISOString(), parentServerId: ids.host }).returning({ id: servers.id }))[0].id;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
describe("servers: tags", () => {
test("a tag is added to those that lack it; those that have it and those that have gone are listed with the reason; undo puts the tags back exactly", async () => {
const r = await send("/servers/bulk", { action: "add_tag", ids: [ids.web, ids.api, ids.db, 99999], tag: "Nightly" });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 3);
assert.deepEqual(await tagsOf(ids.web), ["lab", "nightly"], "written the way tags are");
assert.deepEqual(await tagsOf(ids.db), ["nightly"]);
assert.deepEqual(r.body.undo.items.map((i: { id: number; tags: string[] }) => [i.id, i.tags]).sort(), [[ids.web, ["lab"]], [ids.api, ["lab", "prod"]], [ids.db, []]].sort());
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it no longer exists"]);
const again = await send("/servers/bulk", { action: "add_tag", ids: [ids.web, ids.db], tag: "nightly" });
assert.equal(again.body.changed, 0);
assert.equal(again.body.undo, null, "nothing changed, so nothing to undo");
assert.deepEqual(again.body.skipped.map((s: { reason: string }) => s.reason), ["it already has that tag", "it already has that tag"]);
const undone = await send("/servers/bulk", r.body.undo);
assert.equal(undone.body.changed, 3);
assert.deepEqual([await tagsOf(ids.web), await tagsOf(ids.api), await tagsOf(ids.db)], [["lab"], ["lab", "prod"], []]);
assert.equal((await db.select().from(servers).where(eq(servers.id, ids.db)))[0].tags, null, "no tags is none, not an empty list");
});
test("a tag is taken off those that have it", async () => {
const r = await send("/servers/bulk", { action: "remove_tag", ids: [ids.web, ids.api, ids.db], tag: "lab" });
assert.equal(r.body.changed, 2);
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it doesn't have that tag"]);
assert.deepEqual([await tagsOf(ids.web), await tagsOf(ids.api)], [[], ["prod"]]);
await send("/servers/bulk", r.body.undo);
assert.deepEqual([await tagsOf(ids.web), await tagsOf(ids.api)], [["lab"], ["lab", "prod"]]);
});
test("a wrong tag is refused with the reason; a server with no room for another is skipped; the rest are done", async () => {
assert.equal((await send("/servers/bulk", { action: "add_tag", ids: [ids.web], tag: "bad tag!" })).status, 400);
const full = Array.from({ length: 12 }, (_, i) => `t${String(i).padStart(2, "0")}`);
await db.update(servers).set({ tags: JSON.stringify(full) }).where(eq(servers.id, ids.host));
const r = await send("/servers/bulk", { action: "add_tag", ids: [ids.host, ids.db], tag: "extra" });
assert.equal(r.body.changed, 1);
assert.match(r.body.skipped[0].reason, /at most 12 tags/);
await send("/servers/bulk", r.body.undo);
await db.update(servers).set({ tags: null }).where(eq(servers.id, ids.host));
});
});
describe("servers: maintenance", () => {
test("each is put under maintenance, one already under it is extended; undo ends the new and puts the old back as it was", async () => {
const T = new Date();
await db.delete(maintenanceWindows);
const before = (await db.insert(maintenanceWindows).values({ targetType: "server", targetId: ids.db, reason: "earlier", startedAt: new Date(T.getTime() - 60_000).toISOString(), endsAt: new Date(T.getTime() + 10 * 60_000).toISOString() }).returning())[0];
const r = await send("/servers/bulk", { action: "silence", ids: [ids.web, ids.db], minutes: 90, reason: "Patching" });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 2);
assert.deepEqual(r.body.silenced.map((s: { name: string }) => s.name).sort(), ["db", "web"]);
const during = await listActiveWindows();
assert.equal(during.some((w) => w.targetId === ids.web), true);
assert.ok(Date.parse(during.find((w) => w.targetId === ids.db)!.endsAt) > T.getTime() + 60 * 60_000, "extended to ninety minutes");
assert.equal(r.body.undo.created.length, 1);
assert.deepEqual(r.body.undo.restored.map((x: { id: number; endsAt: string }) => [x.id, x.endsAt]), [[before.id, before.endsAt]]);
const undone = await send("/servers/bulk", r.body.undo);
assert.equal(undone.body.changed, 2);
const after = await listActiveWindows();
assert.equal(after.some((w) => w.targetId === ids.web), false, "the one it made is over");
const old = (await db.select().from(maintenanceWindows).where(eq(maintenanceWindows.id, before.id)))[0];
assert.deepEqual([old.endsAt, old.reason], [before.endsAt, "earlier"], "the one that was there is as it was");
});
test("what depends on a server can be silenced with it", async () => {
await db.delete(maintenanceWindows);
const r = await send("/servers/bulk", { action: "silence", ids: [ids.host], minutes: 30, withDependents: true });
assert.deepEqual(r.body.silenced.map((s: { name: string }) => s.name).sort(), ["guest", "host"]);
assert.match((await listActiveWindows()).find((w) => w.targetId === ids.guest)!.reason ?? "", /^Depends on host/);
await send("/servers/bulk", r.body.undo);
assert.equal((await listActiveWindows()).length, 0);
});
test("how long must be sensible", async () => {
assert.equal((await send("/servers/bulk", { action: "silence", ids: [ids.web], minutes: 1 })).status, 400);
assert.equal((await send("/servers/bulk", { action: "silence", ids: [ids.web], minutes: 99999 })).status, 400);
});
});
describe("servers: which server another runs on", () => {
test("an administrator sets it for several; a loop is skipped; undo restores each exactly; an operator may not", async () => {
who = "operator";
assert.equal((await send("/servers/bulk", { action: "set_parent", ids: [ids.web], parentServerId: ids.host })).status, 403);
who = "admin";
const r = await send("/servers/bulk", { action: "set_parent", ids: [ids.web, ids.api, ids.host], parentServerId: ids.host });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 2);
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["a server can't run on itself"]);
const parent = async (id: number) => (await db.select().from(servers).where(eq(servers.id, id)))[0].parentServerId;
assert.deepEqual([await parent(ids.web), await parent(ids.api)], [ids.host, ids.host]);
// guest already runs on host: making host run on guest would close a loop
const loop = await send("/servers/bulk", { action: "set_parent", ids: [ids.host], parentServerId: ids.guest });
assert.equal(loop.body.changed, 0);
assert.match(loop.body.skipped[0].reason, /each would be waiting for the other/);
assert.equal((await send("/servers/bulk", { action: "set_parent", ids: [ids.web], parentServerId: 99999 })).status, 400);
await send("/servers/bulk", r.body.undo);
assert.deepEqual([await parent(ids.web), await parent(ids.api)], [null, null]);
const auto = await send("/servers/bulk", { action: "set_parent", ids: [ids.web], parentServerId: null });
assert.equal(auto.body.changed, 1, "null goes back to working it out");
who = "operator";
});
test("a viewer can do none of it", async () => {
who = "viewer";
assert.equal((await send("/servers/bulk", { action: "add_tag", ids: [ids.web], tag: "x" })).status, 403);
assert.equal((await send("/servers/bulk", { action: "silence", ids: [ids.web], minutes: 30 })).status, 403);
who = "operator";
});
test("too many, none, or something that isn't an action are refused", async () => {
assert.equal((await send("/servers/bulk", { action: "add_tag", ids: [], tag: "x" })).status, 400);
assert.equal((await send("/servers/bulk", { action: "add_tag", ids: Array.from({ length: 201 }, (_, i) => i + 1), tag: "x" })).status, 400);
assert.equal((await send("/servers/bulk", { action: "delete_everything", ids: [1] })).status, 400);
});
});
describe("alerts", () => {
const alertIds: string[] = [];
before(async () => {
await db.delete(maintenanceWindows);
// three late heartbeats: two on servers (so each has something to silence, and both on the same one), one on nothing
const old = "2026-01-01 00:00:00";
const mk = async (name: string, serverId: number | null) => (await db.insert(heartbeats).values({ name, token: `hb_${name}`.padEnd(24, "x"), periodMinutes: 60, graceMinutes: 5, serverId, createdAt: old }).returning())[0];
const h1 = await mk("job1", ids.web);
const h2 = await mk("job2", ids.web);
const h3 = await mk("job3", null);
invalidateAlertsCache();
const report = await getAlerts(true);
for (const h of [h1, h2, h3]) alertIds.push(report.alerts.find((a) => a.message.startsWith(h.name))!.id);
});
test("several are acknowledged the same way; one that has gone is skipped; undo takes them off again", async () => {
const r = await send("/alerts/bulk", { action: "acknowledge", alertIds: [...alertIds, "automation:heartbeat:99999"], note: "Known, fixing tonight", hours: 12 });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 3);
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it isn't there any more"]);
const rows = await db.select().from(alertAcks);
assert.equal(rows.length, 3);
assert.ok(rows.every((x) => x.note === "Known, fixing tonight" && x.ackedBy === "Olga" && x.until !== null));
assert.ok(r.body.undo.items.every((i: { previous: unknown }) => i.previous === null));
await send("/alerts/bulk", r.body.undo);
assert.equal((await db.select().from(alertAcks)).length, 0);
});
test("an earlier acknowledgement is put back exactly — its note, who it was handed to and when it ends", async () => {
await send("/alerts/bulk", { action: "acknowledge", alertIds: [alertIds[0]], note: "first", hours: 5 });
const [first] = await db.select().from(alertAcks);
const over = await send("/alerts/bulk", { action: "acknowledge", alertIds: [alertIds[0], alertIds[1]], note: "second" });
assert.equal(over.body.changed, 2);
assert.equal((await db.select().from(alertAcks).where(eq(alertAcks.alertKey, alertIds[0])))[0].note, "second");
await send("/alerts/bulk", over.body.undo);
const rows = await db.select().from(alertAcks);
assert.equal(rows.length, 1, "the second alert had none before");
assert.deepEqual([rows[0].note, rows[0].ackedAt, rows[0].until, rows[0].ackedBy], [first.note, first.ackedAt, first.until, first.ackedBy]);
});
test("acknowledgements are taken off, those with none are skipped, and undo puts them back", async () => {
await db.delete(alertAcks);
await send("/alerts/bulk", { action: "acknowledge", alertIds: [alertIds[0], alertIds[1]], note: "mine" });
const r = await send("/alerts/bulk", { action: "unacknowledge", alertIds });
assert.equal(r.body.changed, 2);
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["nobody had acknowledged it"]);
assert.equal((await db.select().from(alertAcks)).length, 0);
await send("/alerts/bulk", r.body.undo);
assert.deepEqual((await db.select().from(alertAcks)).map((x) => x.note), ["mine", "mine"]);
await db.delete(alertAcks);
});
test("handing them to someone who cannot act is refused, and nothing changes", async () => {
const r = await send("/alerts/bulk", { action: "acknowledge", alertIds, assigneeId: (await db.select().from(users).where(eq(users.oidcSub, "viewer")))[0].id });
assert.equal(r.status, 400);
assert.equal((await db.select().from(alertAcks)).length, 0);
});
test("silencing puts each distinct target under maintenance once; an alert with nothing to silence is skipped; undo ends it", async () => {
await db.delete(maintenanceWindows);
const r = await send("/alerts/bulk", { action: "silence", alertIds, minutes: 60, reason: "Weekend" });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 1, "two alerts, one server");
assert.equal(r.body.silenced[0].name, "web");
assert.equal(r.body.skipped.length, 1);
assert.match(r.body.skipped[0].reason, /nothing to put under maintenance/);
assert.equal((await listActiveWindows()).length, 1);
await send("/alerts/bulk", r.body.undo);
assert.equal((await listActiveWindows()).length, 0);
});
test("an operator is needed; a viewer is not enough", async () => {
who = "viewer";
assert.equal((await send("/alerts/bulk", { action: "acknowledge", alertIds })).status, 403);
who = "operator";
});
});
describe("hardware", () => {
const hw: Record<string, number> = {};
const row = async (id: number) => (await db.select().from(assets).where(eq(assets.id, id)))[0];
before(async () => {
const now = new Date().toISOString();
for (const [name, location, warrantyUntil] of [["NAS", "Rack A", "2027-01-01"], ["Switch", null, null], ["UPS", "Hall", "2026-12-01"]] as const) {
hw[name] = (await db.insert(assets).values({ name, kind: "other", location, warrantyUntil, createdAt: now, updatedAt: now } as never).returning({ id: assets.id }))[0].id;
}
});
test("a location is set on several at once, a blank clears it, and undo puts each one's own back", async () => {
const r = await send("/inventory/bulk", { action: "change", ids: [hw.NAS, hw.Switch, hw.UPS, 99999], location: "Closet" });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 3);
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it no longer exists"]);
assert.deepEqual([(await row(hw.NAS)).location, (await row(hw.Switch)).location], ["Closet", "Closet"]);
await send("/inventory/bulk", r.body.undo);
assert.deepEqual([(await row(hw.NAS)).location, (await row(hw.Switch)).location, (await row(hw.UPS)).location], ["Rack A", null, "Hall"]);
const cleared = await send("/inventory/bulk", { action: "change", ids: [hw.NAS], location: " " });
assert.equal((await row(hw.NAS)).location, null);
await send("/inventory/bulk", cleared.body.undo);
assert.equal((await row(hw.NAS)).location, "Rack A");
});
test("the end of support and the warranty are set together or one at a time; whatever is not named is left alone; undo restores", async () => {
const r = await send("/inventory/bulk", { action: "change", ids: [hw.NAS, hw.Switch], endOfSupport: "2029-06-30" });
assert.equal(r.body.changed, 2);
const nas = await row(hw.NAS);
assert.deepEqual([nas.endOfSupport, nas.warrantyUntil, nas.location], ["2029-06-30", "2027-01-01", "Rack A"]);
await send("/inventory/bulk", { action: "change", ids: [hw.NAS], warrantyUntil: null });
assert.equal((await row(hw.NAS)).warrantyUntil, null);
await send("/inventory/bulk", r.body.undo);
assert.equal((await row(hw.NAS)).endOfSupport, null);
assert.equal((await row(hw.Switch)).endOfSupport, null);
});
test("a date that is not a date, or nothing to change, is refused and nothing changes", async () => {
assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS], endOfSupport: "2029-02-30" })).status, 400);
assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS], warrantyUntil: "soon" })).status, 400);
assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS] })).status, 400);
assert.equal((await row(hw.NAS)).location, "Rack A");
});
test("an operator is needed", async () => {
who = "viewer";
assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS], location: "x" })).status, 403);
who = "operator";
});
});
describe("what is written down", () => {
test("each bulk action is one audit entry saying what and how many, and the change log reads them as sentences", async () => {
const entries = await db.select().from(auditLog);
const actions = new Set(entries.map((e) => `${e.category}/${e.action}`));
for (const a of ["server/bulk_tag", "maintenance/bulk_start", "maintenance/bulk_undo", "server/bulk_runs_on", "alert/bulk_ack", "alert/bulk_unack", "alert/bulk_undo", "inventory/bulk_change"]) assert.ok(actions.has(a), a);
const tag = entries.find((e) => e.action === "bulk_tag" && JSON.parse(e.detail!).change === "add_tag")!;
assert.equal(JSON.parse(tag.detail!).tag, "Nightly");
assert.equal(JSON.parse(tag.detail!).changed, 3);
const log = await getChangeLog({ days: 1, limit: 500 });
const titles = log.entries.map((e) => e.title);
assert.ok(titles.some((t) => /^Tag added to 3 servers: Nightly$/.test(t)), "tag");
assert.ok(titles.some((t) => /^Maintenance started on several at once/.test(t)), "maintenance");
assert.ok(titles.some((t) => /alerts? acknowledged at once$/.test(t)), "ack");
assert.ok(titles.some((t) => /pieces? of hardware changed at once$/.test(t)), "hardware");
assert.ok(titles.some((t) => /was undone$/.test(t)), "undo");
});
});