Files
Homelab-manager/server/test/alertClose.test.ts
T
bobbanandClaude Sonnet 5.5 65214f6f0f Nextcloud cron and Nextcloud's own checks; bulk close and open again
Nextcloud: its own "Cron last run" setup check (GET /settings/setupcheck,
Nextcloud 28+, found by class name) raises an alert when background jobs have
stopped; other checks that report an error are one alert, warnings are shown
on the page. The core version Nextcloud says is available is shown too. Before
28, cron and the checks are unknown, never "fine".

Alerts page: bulk Close... (one reason, written on each incident, undoable) and
Open again. Bulk Wake now leaves closed alerts alone.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-25 22:12:10 +01:00

303 lines
18 KiB
TypeScript

/**
* Closing an alert: hidden with no end, until it gets worse or has been gone for half an hour; why it was closed goes on its incident; it can be opened again; and how it sits
* beside snoozing.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-close-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, heartbeats, alertSnoozes, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { alertsRouter } = await import("../src/routes/alerts.js");
const { alertsBulkRouter } = await import("../src/routes/bulk.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { setClosed, setSnooze, settleSnoozes, withoutSnoozed, loadSnoozes, GONE_AFTER_MS, CLOSED_UNTIL } = await import("../src/services/alertSnoozes.js");
const { getIncident, openIncidentIds } = await import("../src/services/incidents.js");
const { getChangeLog } = await import("../src/services/changeLog.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
let app: Server;
let base = "";
let who = "operator";
const call = async (method: string, path: string, body?: unknown) => {
const res = await fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
return { status: res.status, body: (await res.json().catch(() => null)) as any };
};
const idOfHeartbeat = async (name: string) => {
invalidateAlertsCache();
const report = await getAlerts(true);
const a = report.alerts.find((x) => x.message.startsWith(name));
assert.ok(a, `an alert about ${name}`);
return a!;
};
before(async () => {
await db.insert(users).values([
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
const old = "2026-01-01 00:00:00";
for (const name of ["jobA", "jobB"]) await db.insert(heartbeats).values({ name, token: `hb_${name}`.padEnd(24, "x"), periodMinutes: 60, graceMinutes: 5, serverId: null, createdAt: old });
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/alerts/bulk", alertsBulkRouter);
a.use("/api/alerts", alertsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/alerts`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
describe("closing through the route", () => {
test("a closed alert is held back like a snoozed one, with who closed it and why; it is not counted as needing attention", async () => {
const a = await idOfHeartbeat("jobA");
assert.equal(a.silenced, false);
const attention = (c: { critical: number; warning: number; info: number }) => c.critical + c.warning + c.info;
const before = (await getAlerts(false)).counts;
const r = await call("PUT", "/close", { alertId: a.id, note: " Replaced the job; the old one is retired. " });
assert.equal(r.status, 200);
assert.deepEqual([r.body.closed.by, r.body.closed.note], ["Olga", "Replaced the job; the old one is retired."]);
invalidateAlertsCache();
const report = await getAlerts(true);
const after = report.alerts.find((x) => x.id === a.id)!;
assert.deepEqual([after.silenced, after.closed?.by, after.closed?.note, after.snoozed], [true, "Olga", "Replaced the job; the old one is retired.", undefined]);
assert.equal(attention(report.counts), attention(before) - 1, "no longer among those that need a look");
assert.equal(report.counts.silenced, before.silenced + 1, "counted as held back");
});
test("the reason is written on the incident, and opening it again says so there too", async () => {
const a = await idOfHeartbeat("jobA");
const incidentId = (await openIncidentIds([a.id])).get(a.id)!;
assert.ok(incidentId, "an incident is open for it");
let notes = (await getIncident(incidentId))!.notes.map((n) => [n.author, n.body]);
assert.deepEqual(notes, [["Olga", "Closed: Replaced the job; the old one is retired."]]);
assert.equal((await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`)).status, 204);
notes = (await getIncident(incidentId))!.notes.map((n) => [n.author, n.body]);
assert.deepEqual(notes.at(-1), ["Olga", "Opened again."]);
invalidateAlertsCache();
const again = (await getAlerts(true)).alerts.find((x) => x.id === a.id)!;
assert.deepEqual([again.silenced, again.closed], [false, undefined], "shown and counted again");
});
test("it is in the audit log and the change log as sentences; closing without a note says so on the incident", async () => {
const a = await idOfHeartbeat("jobB");
assert.equal((await call("PUT", "/close", { alertId: a.id })).status, 200);
const incidentId = (await openIncidentIds([a.id])).get(a.id)!;
assert.deepEqual((await getIncident(incidentId))!.notes.map((n) => n.body), ["Closed without a note."]);
await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`);
const actions = (await db.select().from(auditLog)).map((e) => `${e.category}/${e.action}`);
assert.ok(actions.includes("alert/close") && actions.includes("alert/reopen"));
const titles = (await getChangeLog({ days: 1, limit: 300 })).entries.map((e) => e.title);
assert.ok(titles.some((t) => /^Alert closed: jobA/.test(t)), "closed");
assert.ok(titles.some((t) => /^Closed alert opened again: jobA/.test(t)), "opened again");
});
test("an alert that is not there is not found, a note that is too long is refused, a viewer cannot close or open", async () => {
assert.equal((await call("PUT", "/close", { alertId: "monitoring:nothing:1" })).status, 404);
const a = await idOfHeartbeat("jobA");
const long = await call("PUT", "/close", { alertId: a.id, note: "x".repeat(501) });
assert.equal(long.status, 400);
who = "viewer";
assert.equal((await call("PUT", "/close", { alertId: a.id })).status, 403);
assert.equal((await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`)).status, 403);
who = "operator";
assert.equal((await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`)).status, 404, "nothing to open");
});
test("closing an alert that is snoozed makes it closed, and snoozing a closed one makes it a snooze again", async () => {
const a = await idOfHeartbeat("jobA");
const until = new Date(Date.now() + 3_600_000).toISOString();
assert.equal((await call("PUT", "/snooze", { alertId: a.id, until })).status, 200);
assert.equal((await call("PUT", "/close", { alertId: a.id })).status, 200);
let [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id));
assert.deepEqual([row.kind, row.until], ["closed", CLOSED_UNTIL]);
assert.equal((await call("PUT", "/snooze", { alertId: a.id, until })).status, 200);
[row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id));
assert.deepEqual([row.kind, row.severity, row.note], ["snooze", null, null]);
await call("DELETE", `/snooze?alertId=${encodeURIComponent(a.id)}`);
});
test("a bulk snooze over a closed alert can be undone back to closed, with its note", async () => {
const a = await idOfHeartbeat("jobA");
await call("PUT", "/close", { alertId: a.id, note: "accepted" });
const r = await call("POST", "/bulk", { action: "snooze", alertIds: [a.id], until: new Date(Date.now() + 3_600_000).toISOString() });
assert.equal(r.body.changed, 1);
assert.equal((await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id)))[0].kind, "snooze");
await call("POST", "/bulk", r.body.undo);
const [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id));
assert.deepEqual([row.kind, row.note, row.until], ["closed", "accepted", CLOSED_UNTIL]);
await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`);
});
});
describe("until it gets worse", () => {
const present = (id: string, severity: string, message = "disk is 91% full") => [{ id, message, severity, key: `k:${id}` }];
const NOW = new Date();
test("the same alert stays closed however its message changes, and a milder severity does not open it", async () => {
await db.delete(alertSnoozes);
await setClosed({ alertId: "x1", conditionKey: "k:x1", message: "disk is 91% full", severity: "warning", note: null, actor: "Olga" }, NOW);
assert.deepEqual(await settleSnoozes(present("x1", "warning", "disk is 94% full"), NOW), []);
assert.deepEqual(await settleSnoozes(present("x1", "info", "disk is 80% full"), NOW), []);
assert.equal((await db.select().from(alertSnoozes)).length, 1, "still closed");
assert.equal((await loadSnoozes(NOW)).get("x1")?.closed, true);
});
test("a higher severity opens it again and says so, once", async () => {
const reminders = await settleSnoozes(present("x1", "critical", "disk is 97% full"), NOW);
assert.deepEqual(reminders.map((r) => [r.key, r.reason, r.severity, r.message]), [["k:x1", "worse", "critical", "disk is 97% full"]]);
assert.equal((await db.select().from(alertSnoozes)).length, 0);
assert.deepEqual(await settleSnoozes(present("x1", "critical"), NOW), [], "nothing more to say");
});
test("gone for less than half an hour it stays closed; gone for longer it is forgotten, so the same problem later is new", async () => {
await db.delete(alertSnoozes);
await setClosed({ alertId: "x2", conditionKey: "k:x2", message: "m", severity: "warning", note: null, actor: "Olga" }, NOW);
await settleSnoozes([], new Date(NOW.getTime() + GONE_AFTER_MS - 60_000));
assert.equal((await db.select().from(alertSnoozes)).length, 1);
await settleSnoozes([], new Date(NOW.getTime() + GONE_AFTER_MS + 60_000));
assert.equal((await db.select().from(alertSnoozes)).length, 0);
});
test("a closed alert never expires like a snooze does, however much time passes while it is there", async () => {
await db.delete(alertSnoozes);
await setClosed({ alertId: "x3", conditionKey: "k:x3", message: "m", severity: "warning", note: null, actor: "Olga" }, NOW);
const tenYears = new Date(NOW.getTime() + 10 * 365 * 86_400_000);
assert.deepEqual(await settleSnoozes(present("x3", "warning"), tenYears), []);
assert.equal((await loadSnoozes(tenYears)).has("x3"), true);
});
test("messages about a closed alert are held back like a snoozed one's, and a snooze that ends still reminds as before", async () => {
await db.delete(alertSnoozes);
await setClosed({ alertId: "x4", conditionKey: "k:x4", message: "m", severity: "warning", note: null, actor: "Olga" }, NOW);
await setSnooze({ alertId: "x5", conditionKey: "k:x5", message: "snoozed one", until: new Date(NOW.getTime() + 120_000), actor: "Olga" }, NOW);
assert.deepEqual((await withoutSnoozed([{ key: "k:x4" }, { key: "k:other" }, { key: "k:x5" }], NOW)).map((i) => i.key), ["k:other"]);
const reminders = await settleSnoozes(present("x5", "warning", "snoozed one"), new Date(NOW.getTime() + 180_000));
assert.deepEqual(reminders.map((r) => [r.key, r.reason]), [["k:x5", "ended"]]);
});
});
describe("closing several at once", () => {
const notesOf = async (alertId: string) => {
const incidentId = (await openIncidentIds([alertId])).get(alertId)!;
return (await getIncident(incidentId))!.notes.map((n) => n.body);
};
const clear = async () => {
await db.delete(alertSnoozes);
};
test("several are closed with one reason; each incident says so; the one that has gone is skipped; they are held back", async () => {
await clear();
const a = await idOfHeartbeat("jobA");
const b = await idOfHeartbeat("jobB");
const r = await call("POST", "/bulk", { action: "close", alertIds: [a.id, b.id, "monitoring:nothing:1"], note: " Both retired with the old host. " });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 2);
assert.deepEqual(r.body.skipped.map((x: { reason: string }) => x.reason), ["it isn't there any more"]);
const rows = await db.select().from(alertSnoozes);
assert.deepEqual(rows.map((x) => [x.kind, x.note, x.snoozedBy]), [["closed", "Both retired with the old host.", "Olga"], ["closed", "Both retired with the old host.", "Olga"]]);
assert.deepEqual(rows.map((x) => x.severity), [a.severity, b.severity]);
assert.ok((await notesOf(a.id)).includes("Closed: Both retired with the old host. (closed together with 1 other)"));
invalidateAlertsCache();
const report = await getAlerts(true);
assert.ok([a.id, b.id].every((id) => report.alerts.find((x) => x.id === id)?.closed?.by === "Olga"));
await call("POST", "/bulk", r.body.undo);
});
test("undo opens them again and says so on each incident; an alert that was closed before the bulk keeps its own closing and note", async () => {
await clear();
const a = await idOfHeartbeat("jobA");
const b = await idOfHeartbeat("jobB");
await call("PUT", "/close", { alertId: a.id, note: "mine from before" });
const r = await call("POST", "/bulk", { action: "close", alertIds: [a.id, b.id], note: "bulk" });
assert.equal(r.body.changed, 2);
assert.equal(r.body.undo.closed, true);
assert.equal((await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id)))[0].note, "bulk", "replaced");
const undone = (id: string) => notesOf(id).then((n) => n.filter((x) => x === "Opened again (closing it was undone).").length);
const [beforeA, beforeB] = [await undone(a.id), await undone(b.id)];
await call("POST", "/bulk", r.body.undo);
const rows = await db.select().from(alertSnoozes);
assert.deepEqual(rows.map((x) => [x.alertId, x.note]), [[a.id, "mine from before"]], "a is closed as it was; b is open again");
assert.equal(await undone(b.id), beforeB + 1, "b is open again and says so");
assert.equal(await undone(a.id), beforeA, "a was not opened by the undo, so says nothing");
await clear();
});
test("open again: only closed alerts are opened, one that is merely snoozed is left as it is, and the incident says so; undo closes them again", async () => {
await clear();
const a = await idOfHeartbeat("jobA");
const b = await idOfHeartbeat("jobB");
await call("PUT", "/close", { alertId: a.id, note: "accepted" });
await call("PUT", "/snooze", { alertId: b.id, until: new Date(Date.now() + 3_600_000).toISOString() });
const r = await call("POST", "/bulk", { action: "reopen", alertIds: [a.id, b.id] });
assert.equal(r.body.changed, 1);
assert.deepEqual(r.body.skipped.map((x: { reason: string }) => x.reason), ["it wasn't closed"]);
assert.deepEqual((await db.select().from(alertSnoozes)).map((x) => [x.alertId, x.kind]), [[b.id, "snooze"]]);
assert.equal((await notesOf(a.id)).at(-1), "Opened again.");
await call("POST", "/bulk", r.body.undo);
const back = (await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id)))[0];
assert.deepEqual([back.kind, back.note], ["closed", "accepted"]);
await clear();
});
test("waking snoozed alerts leaves a closed one alone and says to use Open again", async () => {
await clear();
const a = await idOfHeartbeat("jobA");
await call("PUT", "/close", { alertId: a.id });
const r = await call("POST", "/bulk", { action: "unsnooze", alertIds: [a.id] });
assert.equal(r.body.changed, 0);
assert.match(r.body.skipped[0].reason, /closed, not snoozed — use Open again/);
assert.equal((await db.select().from(alertSnoozes)).length, 1, "still closed");
await clear();
});
test("a note that is too long is refused and nothing is closed; a viewer cannot; both are in the audit log and the change log", async () => {
await clear();
const a = await idOfHeartbeat("jobA");
const long = await call("POST", "/bulk", { action: "close", alertIds: [a.id], note: "x".repeat(501) });
assert.equal(long.status, 400);
assert.equal((await db.select().from(alertSnoozes)).length, 0);
who = "viewer";
assert.equal((await call("POST", "/bulk", { action: "close", alertIds: [a.id] })).status, 403);
assert.equal((await call("POST", "/bulk", { action: "reopen", alertIds: [a.id] })).status, 403);
who = "operator";
await call("POST", "/bulk", { action: "close", alertIds: [a.id], note: "n" });
await call("POST", "/bulk", { action: "reopen", alertIds: [a.id] });
const actions = (await db.select().from(auditLog)).map((e) => `${e.category}/${e.action}`);
assert.ok(actions.includes("alert/bulk_close") && actions.includes("alert/bulk_reopen"));
const titles = (await getChangeLog({ days: 1, limit: 400 })).entries.map((e) => e.title);
assert.ok(titles.some((t) => /alerts? closed at once$/.test(t)), "close");
assert.ok(titles.some((t) => /closed alerts? opened again at once$/.test(t)), "reopen");
assert.ok(titles.some((t) => /^Closing several alerts at once was undone$/.test(t)), "undo");
await clear();
});
});