/** * Bulk actions: the same thing done to several alerts, servers or hardware entries, who may do which, what is skipped and why, and that the body each one hands back * puts everything exactly as it was. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; import type { Server } from "node:http"; const dir = mkdtempSync(join(tmpdir(), "hm-bulk-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers, heartbeats, assets, alertAcks, maintenanceWindows, auditLog } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { serversBulkRouter, alertsBulkRouter, inventoryBulkRouter } = await import("../src/routes/bulk.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { listActiveWindows } = await import("../src/services/maintenance.js"); const { getChangeLog } = await import("../src/services/changeLog.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); let app: Server; let base = ""; let who = "operator"; const send = async (path: string, body: unknown) => { const res = await fetch(`${base}${path}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); return { status: res.status, body: (await res.json().catch(() => null)) as any }; }; const tagsOf = async (id: number) => JSON.parse((await db.select().from(servers).where(eq(servers.id, id)))[0].tags ?? "[]") as string[]; const ids: Record = {}; before(async () => { await db.insert(users).values([ { oidcSub: "admin", name: "Ada", role: "admin" }, { oidcSub: "operator", name: "Olga", role: "operator" }, { oidcSub: "viewer", name: "Vic", role: "viewer" }, ]); const a = express(); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/servers/bulk", serversBulkRouter); a.use("/api/alerts/bulk", alertsBulkRouter); a.use("/api/inventory/bulk", inventoryBulkRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`; for (const [name, tags] of [["web", ["lab"]], ["api", ["lab", "prod"]], ["db", null], ["host", null]] as const) { ids[name] = (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, tags: tags ? JSON.stringify(tags) : null, lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id; } // a guest that runs on host: what depends on it ids.guest = (await db.insert(servers).values({ name: "guest", apiTokenHash: "g", apiTokenPrefix: "g", lastSeenAt: new Date().toISOString(), parentServerId: ids.host }).returning({ id: servers.id }))[0].id; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); describe("servers: tags", () => { test("a tag is added to those that lack it; those that have it and those that have gone are listed with the reason; undo puts the tags back exactly", async () => { const r = await send("/servers/bulk", { action: "add_tag", ids: [ids.web, ids.api, ids.db, 99999], tag: "Nightly" }); assert.equal(r.status, 200); assert.equal(r.body.changed, 3); assert.deepEqual(await tagsOf(ids.web), ["lab", "nightly"], "written the way tags are"); assert.deepEqual(await tagsOf(ids.db), ["nightly"]); assert.deepEqual(r.body.undo.items.map((i: { id: number; tags: string[] }) => [i.id, i.tags]).sort(), [[ids.web, ["lab"]], [ids.api, ["lab", "prod"]], [ids.db, []]].sort()); assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it no longer exists"]); const again = await send("/servers/bulk", { action: "add_tag", ids: [ids.web, ids.db], tag: "nightly" }); assert.equal(again.body.changed, 0); assert.equal(again.body.undo, null, "nothing changed, so nothing to undo"); assert.deepEqual(again.body.skipped.map((s: { reason: string }) => s.reason), ["it already has that tag", "it already has that tag"]); const undone = await send("/servers/bulk", r.body.undo); assert.equal(undone.body.changed, 3); assert.deepEqual([await tagsOf(ids.web), await tagsOf(ids.api), await tagsOf(ids.db)], [["lab"], ["lab", "prod"], []]); assert.equal((await db.select().from(servers).where(eq(servers.id, ids.db)))[0].tags, null, "no tags is none, not an empty list"); }); test("a tag is taken off those that have it", async () => { const r = await send("/servers/bulk", { action: "remove_tag", ids: [ids.web, ids.api, ids.db], tag: "lab" }); assert.equal(r.body.changed, 2); assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it doesn't have that tag"]); assert.deepEqual([await tagsOf(ids.web), await tagsOf(ids.api)], [[], ["prod"]]); await send("/servers/bulk", r.body.undo); assert.deepEqual([await tagsOf(ids.web), await tagsOf(ids.api)], [["lab"], ["lab", "prod"]]); }); test("a wrong tag is refused with the reason; a server with no room for another is skipped; the rest are done", async () => { assert.equal((await send("/servers/bulk", { action: "add_tag", ids: [ids.web], tag: "bad tag!" })).status, 400); const full = Array.from({ length: 12 }, (_, i) => `t${String(i).padStart(2, "0")}`); await db.update(servers).set({ tags: JSON.stringify(full) }).where(eq(servers.id, ids.host)); const r = await send("/servers/bulk", { action: "add_tag", ids: [ids.host, ids.db], tag: "extra" }); assert.equal(r.body.changed, 1); assert.match(r.body.skipped[0].reason, /at most 12 tags/); await send("/servers/bulk", r.body.undo); await db.update(servers).set({ tags: null }).where(eq(servers.id, ids.host)); }); }); describe("servers: maintenance", () => { test("each is put under maintenance, one already under it is extended; undo ends the new and puts the old back as it was", async () => { const T = new Date(); await db.delete(maintenanceWindows); const before = (await db.insert(maintenanceWindows).values({ targetType: "server", targetId: ids.db, reason: "earlier", startedAt: new Date(T.getTime() - 60_000).toISOString(), endsAt: new Date(T.getTime() + 10 * 60_000).toISOString() }).returning())[0]; const r = await send("/servers/bulk", { action: "silence", ids: [ids.web, ids.db], minutes: 90, reason: "Patching" }); assert.equal(r.status, 200); assert.equal(r.body.changed, 2); assert.deepEqual(r.body.silenced.map((s: { name: string }) => s.name).sort(), ["db", "web"]); const during = await listActiveWindows(); assert.equal(during.some((w) => w.targetId === ids.web), true); assert.ok(Date.parse(during.find((w) => w.targetId === ids.db)!.endsAt) > T.getTime() + 60 * 60_000, "extended to ninety minutes"); assert.equal(r.body.undo.created.length, 1); assert.deepEqual(r.body.undo.restored.map((x: { id: number; endsAt: string }) => [x.id, x.endsAt]), [[before.id, before.endsAt]]); const undone = await send("/servers/bulk", r.body.undo); assert.equal(undone.body.changed, 2); const after = await listActiveWindows(); assert.equal(after.some((w) => w.targetId === ids.web), false, "the one it made is over"); const old = (await db.select().from(maintenanceWindows).where(eq(maintenanceWindows.id, before.id)))[0]; assert.deepEqual([old.endsAt, old.reason], [before.endsAt, "earlier"], "the one that was there is as it was"); }); test("what depends on a server can be silenced with it", async () => { await db.delete(maintenanceWindows); const r = await send("/servers/bulk", { action: "silence", ids: [ids.host], minutes: 30, withDependents: true }); assert.deepEqual(r.body.silenced.map((s: { name: string }) => s.name).sort(), ["guest", "host"]); assert.match((await listActiveWindows()).find((w) => w.targetId === ids.guest)!.reason ?? "", /^Depends on host/); await send("/servers/bulk", r.body.undo); assert.equal((await listActiveWindows()).length, 0); }); test("how long must be sensible", async () => { assert.equal((await send("/servers/bulk", { action: "silence", ids: [ids.web], minutes: 1 })).status, 400); assert.equal((await send("/servers/bulk", { action: "silence", ids: [ids.web], minutes: 99999 })).status, 400); }); }); describe("servers: which server another runs on", () => { test("an administrator sets it for several; a loop is skipped; undo restores each exactly; an operator may not", async () => { who = "operator"; assert.equal((await send("/servers/bulk", { action: "set_parent", ids: [ids.web], parentServerId: ids.host })).status, 403); who = "admin"; const r = await send("/servers/bulk", { action: "set_parent", ids: [ids.web, ids.api, ids.host], parentServerId: ids.host }); assert.equal(r.status, 200); assert.equal(r.body.changed, 2); assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["a server can't run on itself"]); const parent = async (id: number) => (await db.select().from(servers).where(eq(servers.id, id)))[0].parentServerId; assert.deepEqual([await parent(ids.web), await parent(ids.api)], [ids.host, ids.host]); // guest already runs on host: making host run on guest would close a loop const loop = await send("/servers/bulk", { action: "set_parent", ids: [ids.host], parentServerId: ids.guest }); assert.equal(loop.body.changed, 0); assert.match(loop.body.skipped[0].reason, /each would be waiting for the other/); assert.equal((await send("/servers/bulk", { action: "set_parent", ids: [ids.web], parentServerId: 99999 })).status, 400); await send("/servers/bulk", r.body.undo); assert.deepEqual([await parent(ids.web), await parent(ids.api)], [null, null]); const auto = await send("/servers/bulk", { action: "set_parent", ids: [ids.web], parentServerId: null }); assert.equal(auto.body.changed, 1, "null goes back to working it out"); who = "operator"; }); test("a viewer can do none of it", async () => { who = "viewer"; assert.equal((await send("/servers/bulk", { action: "add_tag", ids: [ids.web], tag: "x" })).status, 403); assert.equal((await send("/servers/bulk", { action: "silence", ids: [ids.web], minutes: 30 })).status, 403); who = "operator"; }); test("too many, none, or something that isn't an action are refused", async () => { assert.equal((await send("/servers/bulk", { action: "add_tag", ids: [], tag: "x" })).status, 400); assert.equal((await send("/servers/bulk", { action: "add_tag", ids: Array.from({ length: 201 }, (_, i) => i + 1), tag: "x" })).status, 400); assert.equal((await send("/servers/bulk", { action: "delete_everything", ids: [1] })).status, 400); }); }); describe("alerts", () => { const alertIds: string[] = []; before(async () => { await db.delete(maintenanceWindows); // three late heartbeats: two on servers (so each has something to silence, and both on the same one), one on nothing const old = "2026-01-01 00:00:00"; const mk = async (name: string, serverId: number | null) => (await db.insert(heartbeats).values({ name, token: `hb_${name}`.padEnd(24, "x"), periodMinutes: 60, graceMinutes: 5, serverId, createdAt: old }).returning())[0]; const h1 = await mk("job1", ids.web); const h2 = await mk("job2", ids.web); const h3 = await mk("job3", null); invalidateAlertsCache(); const report = await getAlerts(true); for (const h of [h1, h2, h3]) alertIds.push(report.alerts.find((a) => a.message.startsWith(h.name))!.id); }); test("several are acknowledged the same way; one that has gone is skipped; undo takes them off again", async () => { const r = await send("/alerts/bulk", { action: "acknowledge", alertIds: [...alertIds, "automation:heartbeat:99999"], note: "Known, fixing tonight", hours: 12 }); assert.equal(r.status, 200); assert.equal(r.body.changed, 3); assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it isn't there any more"]); const rows = await db.select().from(alertAcks); assert.equal(rows.length, 3); assert.ok(rows.every((x) => x.note === "Known, fixing tonight" && x.ackedBy === "Olga" && x.until !== null)); assert.ok(r.body.undo.items.every((i: { previous: unknown }) => i.previous === null)); await send("/alerts/bulk", r.body.undo); assert.equal((await db.select().from(alertAcks)).length, 0); }); test("an earlier acknowledgement is put back exactly — its note, who it was handed to and when it ends", async () => { await send("/alerts/bulk", { action: "acknowledge", alertIds: [alertIds[0]], note: "first", hours: 5 }); const [first] = await db.select().from(alertAcks); const over = await send("/alerts/bulk", { action: "acknowledge", alertIds: [alertIds[0], alertIds[1]], note: "second" }); assert.equal(over.body.changed, 2); assert.equal((await db.select().from(alertAcks).where(eq(alertAcks.alertKey, alertIds[0])))[0].note, "second"); await send("/alerts/bulk", over.body.undo); const rows = await db.select().from(alertAcks); assert.equal(rows.length, 1, "the second alert had none before"); assert.deepEqual([rows[0].note, rows[0].ackedAt, rows[0].until, rows[0].ackedBy], [first.note, first.ackedAt, first.until, first.ackedBy]); }); test("acknowledgements are taken off, those with none are skipped, and undo puts them back", async () => { await db.delete(alertAcks); await send("/alerts/bulk", { action: "acknowledge", alertIds: [alertIds[0], alertIds[1]], note: "mine" }); const r = await send("/alerts/bulk", { action: "unacknowledge", alertIds }); assert.equal(r.body.changed, 2); assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["nobody had acknowledged it"]); assert.equal((await db.select().from(alertAcks)).length, 0); await send("/alerts/bulk", r.body.undo); assert.deepEqual((await db.select().from(alertAcks)).map((x) => x.note), ["mine", "mine"]); await db.delete(alertAcks); }); test("handing them to someone who cannot act is refused, and nothing changes", async () => { const r = await send("/alerts/bulk", { action: "acknowledge", alertIds, assigneeId: (await db.select().from(users).where(eq(users.oidcSub, "viewer")))[0].id }); assert.equal(r.status, 400); assert.equal((await db.select().from(alertAcks)).length, 0); }); test("silencing puts each distinct target under maintenance once; an alert with nothing to silence is skipped; undo ends it", async () => { await db.delete(maintenanceWindows); const r = await send("/alerts/bulk", { action: "silence", alertIds, minutes: 60, reason: "Weekend" }); assert.equal(r.status, 200); assert.equal(r.body.changed, 1, "two alerts, one server"); assert.equal(r.body.silenced[0].name, "web"); assert.equal(r.body.skipped.length, 1); assert.match(r.body.skipped[0].reason, /nothing to put under maintenance/); assert.equal((await listActiveWindows()).length, 1); await send("/alerts/bulk", r.body.undo); assert.equal((await listActiveWindows()).length, 0); }); test("an operator is needed; a viewer is not enough", async () => { who = "viewer"; assert.equal((await send("/alerts/bulk", { action: "acknowledge", alertIds })).status, 403); who = "operator"; }); }); describe("hardware", () => { const hw: Record = {}; const row = async (id: number) => (await db.select().from(assets).where(eq(assets.id, id)))[0]; before(async () => { const now = new Date().toISOString(); for (const [name, location, warrantyUntil] of [["NAS", "Rack A", "2027-01-01"], ["Switch", null, null], ["UPS", "Hall", "2026-12-01"]] as const) { hw[name] = (await db.insert(assets).values({ name, kind: "other", location, warrantyUntil, createdAt: now, updatedAt: now } as never).returning({ id: assets.id }))[0].id; } }); test("a location is set on several at once, a blank clears it, and undo puts each one's own back", async () => { const r = await send("/inventory/bulk", { action: "change", ids: [hw.NAS, hw.Switch, hw.UPS, 99999], location: "Closet" }); assert.equal(r.status, 200); assert.equal(r.body.changed, 3); assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it no longer exists"]); assert.deepEqual([(await row(hw.NAS)).location, (await row(hw.Switch)).location], ["Closet", "Closet"]); await send("/inventory/bulk", r.body.undo); assert.deepEqual([(await row(hw.NAS)).location, (await row(hw.Switch)).location, (await row(hw.UPS)).location], ["Rack A", null, "Hall"]); const cleared = await send("/inventory/bulk", { action: "change", ids: [hw.NAS], location: " " }); assert.equal((await row(hw.NAS)).location, null); await send("/inventory/bulk", cleared.body.undo); assert.equal((await row(hw.NAS)).location, "Rack A"); }); test("the end of support and the warranty are set together or one at a time; whatever is not named is left alone; undo restores", async () => { const r = await send("/inventory/bulk", { action: "change", ids: [hw.NAS, hw.Switch], endOfSupport: "2029-06-30" }); assert.equal(r.body.changed, 2); const nas = await row(hw.NAS); assert.deepEqual([nas.endOfSupport, nas.warrantyUntil, nas.location], ["2029-06-30", "2027-01-01", "Rack A"]); await send("/inventory/bulk", { action: "change", ids: [hw.NAS], warrantyUntil: null }); assert.equal((await row(hw.NAS)).warrantyUntil, null); await send("/inventory/bulk", r.body.undo); assert.equal((await row(hw.NAS)).endOfSupport, null); assert.equal((await row(hw.Switch)).endOfSupport, null); }); test("a date that is not a date, or nothing to change, is refused and nothing changes", async () => { assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS], endOfSupport: "2029-02-30" })).status, 400); assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS], warrantyUntil: "soon" })).status, 400); assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS] })).status, 400); assert.equal((await row(hw.NAS)).location, "Rack A"); }); test("an operator is needed", async () => { who = "viewer"; assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS], location: "x" })).status, 403); who = "operator"; }); }); describe("what is written down", () => { test("each bulk action is one audit entry saying what and how many, and the change log reads them as sentences", async () => { const entries = await db.select().from(auditLog); const actions = new Set(entries.map((e) => `${e.category}/${e.action}`)); for (const a of ["server/bulk_tag", "maintenance/bulk_start", "maintenance/bulk_undo", "server/bulk_runs_on", "alert/bulk_ack", "alert/bulk_unack", "alert/bulk_undo", "inventory/bulk_change"]) assert.ok(actions.has(a), a); const tag = entries.find((e) => e.action === "bulk_tag" && JSON.parse(e.detail!).change === "add_tag")!; assert.equal(JSON.parse(tag.detail!).tag, "Nightly"); assert.equal(JSON.parse(tag.detail!).changed, 3); const log = await getChangeLog({ days: 1, limit: 500 }); const titles = log.entries.map((e) => e.title); assert.ok(titles.some((t) => /^Tag added to 3 servers: Nightly$/.test(t)), "tag"); assert.ok(titles.some((t) => /^Maintenance started on several at once/.test(t)), "maintenance"); assert.ok(titles.some((t) => /alerts? acknowledged at once$/.test(t)), "ack"); assert.ok(titles.some((t) => /pieces? of hardware changed at once$/.test(t)), "hardware"); assert.ok(titles.some((t) => /was undone$/.test(t)), "undo"); }); });