From Dockhand, on the health pass: a container that keeps restarting (critical, with its exit code), one whose own health check fails, one that stopped unexpectedly, one created and never started for an hour, and a Docker host Dockhand cannot reach. A stop is only news when it was seen happen, so stacks switched off long ago stay quiet. A Leave alone button on the Docker page and a list in Settings; on by default. Also fixes a date-dependent storage-forecast test that pinned its clock while the code under test reads the real one. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
328 lines
24 KiB
TypeScript
328 lines
24 KiB
TypeScript
/**
|
|
* Alert rules of your own, forecasts for the pools the integrations report, and snoozing an alert.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { createServer, type Server } from "node:http";
|
|
import type { AddressInfo } from "node:net";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-rss-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const express = (await import("express")).default;
|
|
const { db } = await import("../src/db/client.js");
|
|
const { users, servers, serverMetrics, integrations, alertRules, storageSeries, storageSamples, alertSnoozes, maintenanceWindows } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const { evaluateRules, currentRun, checkedRule, computeRuleFindings, listRules, findingKey, ruleCovers, RuleError } = await import("../src/services/alertRules.js");
|
|
const { evaluateHealth } = await import("../src/services/healthMonitor.js");
|
|
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
|
const { metaOfConditionKey } = await import("../src/services/notifyRouting.js");
|
|
const { poolsOf, recordStorageSamples, storageOverview, storageFillingSoon } = await import("../src/services/storageHistory.js");
|
|
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
|
const { setSnooze, clearSnooze, loadSnoozes, withoutSnoozed, settleSnoozes, SnoozeError } = await import("../src/services/alertSnoozes.js");
|
|
const { alertRulesRouter } = await import("../src/routes/alertRules.js");
|
|
const { alertsRouter } = await import("../src/routes/alerts.js");
|
|
const { storageRouter } = await import("../src/routes/storage.js");
|
|
const { eq } = await import("drizzle-orm");
|
|
|
|
await runMigrations();
|
|
|
|
const MIN = 60_000;
|
|
const DAY = 86_400_000;
|
|
const NOW = Date.parse("2026-10-20T12:00:00Z");
|
|
type Rule = Parameters<typeof evaluateRules>[0][number];
|
|
const rule = (extra: Partial<Rule> = {}): Rule => ({ id: 1, name: "CPU hot", enabled: true, metric: "cpu", mount: null, operator: "above", threshold: 90, forMinutes: 15, severity: "warning", serverId: null, tag: null, createdBy: null, createdAt: "", ...extra });
|
|
const samplesAt = (values: number[], endsAgoMin = 1, gapMin = 15) => values.map((v, i) => ({ t: NOW - (endsAgoMin + (values.length - 1 - i) * gapMin) * MIN, v }));
|
|
const server = (id: number, name: string, tags: string[] = [], smart: { device: string; temperatureC: number | null }[] = []) => ({ id, name, tags, smart });
|
|
|
|
after(() => {
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows may still hold the database file
|
|
}
|
|
});
|
|
|
|
describe("your own alert rules", () => {
|
|
test("it has to have been past the limit for the whole time, on every sample, up to now", () => {
|
|
assert.equal(currentRun(rule(), samplesAt([95]), NOW)?.since, NOW - 1 * MIN);
|
|
const held = currentRun(rule(), samplesAt([95, 96, 97]), NOW)!;
|
|
assert.deepEqual([held.value, (held.latest - held.since) / MIN], [97, 30]);
|
|
assert.equal(currentRun(rule(), samplesAt([95, 60, 97]), NOW)!.since, NOW - 1 * MIN, "a dip starts the count again");
|
|
assert.equal(currentRun(rule(), samplesAt([95, 96, 60]), NOW), null, "the newest sample is fine");
|
|
assert.equal(currentRun(rule(), samplesAt([95, 96, 97], 50), NOW), null, "too old to say what it is now");
|
|
assert.equal(currentRun(rule({ operator: "below", threshold: 10 }), samplesAt([5, 4]), NOW)!.value, 4);
|
|
});
|
|
|
|
test("a rule with a time waits for it; without one it speaks at once; a different server or tag is left alone", () => {
|
|
const s = [server(1, "web", ["prod"]), server(2, "pi", ["lab"])];
|
|
const hot = new Map([["1|cpu", samplesAt([95, 96])], ["2|cpu", samplesAt([95, 96])]]);
|
|
const found = (r: Rule, m = hot) => evaluateRules([r], s, m, NOW).map((f) => f.serverName);
|
|
assert.deepEqual(found(rule()), ["web", "pi"], "30 minutes past the limit is more than the 15 asked");
|
|
assert.deepEqual(found(rule({ forMinutes: 60 })), [], "but not an hour");
|
|
assert.deepEqual(found(rule({ forMinutes: 0 }), new Map([["1|cpu", samplesAt([95])]])), ["web"], "one sample is enough without a time");
|
|
assert.deepEqual(found(rule({ tag: "LAB" })), ["pi"], "a tag, however it is written");
|
|
assert.deepEqual(found(rule({ serverId: 1 })), ["web"]);
|
|
assert.equal(ruleCovers({ serverId: null, tag: null }, { id: 9, tags: [] }), true);
|
|
assert.deepEqual(evaluateRules([rule({ enabled: false })], s, hot, NOW), []);
|
|
});
|
|
|
|
test("a disk is judged mount by mount, or just the one named; a temperature is judged as of the last report, per disk", () => {
|
|
const s = [server(1, "nas", [], [{ device: "/dev/sda", temperatureC: 58 }, { device: "/dev/sdb", temperatureC: 41 }, { device: "/dev/sdc", temperatureC: null }])];
|
|
const disks = new Map([["1|disk:/", samplesAt([88, 92])], ["1|disk:/data", samplesAt([91, 93])], ["1|cpu", samplesAt([99])]]);
|
|
const every = evaluateRules([rule({ metric: "disk", threshold: 90, forMinutes: 0 })], s, disks, NOW);
|
|
assert.deepEqual(every.map((f) => f.detail), ["/", "/data"]);
|
|
assert.match(every[1].message, /nas: \/data use is 93%, above 90% \[CPU hot\]/);
|
|
assert.deepEqual(evaluateRules([rule({ metric: "disk", threshold: 90, forMinutes: 0, mount: "/data" })], s, disks, NOW).map((f) => f.detail), ["/data"]);
|
|
const temp = evaluateRules([rule({ metric: "disk_temp", threshold: 55, forMinutes: 0, severity: "critical", name: "Warm disk" })], s, new Map(), NOW);
|
|
assert.deepEqual(temp.map((f) => [f.detail, f.value, f.severity, f.since]), [["/dev/sda", 58, "critical", null]]);
|
|
assert.match(temp[0].message, /nas: \/dev\/sda is 58 °C, above 55 °C \[Warm disk\]/);
|
|
});
|
|
|
|
test("a finding is a health condition of the server — announced, silenced by maintenance and routed like the rest — unless the server is offline", () => {
|
|
const f = evaluateRules([rule()], [server(1, "web")], new Map([["1|cpu", samplesAt([95, 96])]]), NOW);
|
|
assert.equal(findingKey(f[0]), "rule:1:server:1");
|
|
assert.equal(findingKey({ ruleId: 2, serverId: 4, detail: "/data" }), "rule:2:server:4:/data");
|
|
const snap = (lastSeenAt: string) => ({ servers: [{ id: 1, name: "web", lastSeenAt, disks: [] }], proxmox: [], synology: [], rules: f });
|
|
const thresholds = { serverOfflineMinutes: 60, diskUsagePercent: 90 };
|
|
const ok = evaluateHealth(snap(new Date(NOW - MIN).toISOString()), thresholds, NOW);
|
|
assert.deepEqual(ok.map((c) => [c.key, c.source, c.severity]), [["rule:1:server:1", "server:rules", "warning"]]);
|
|
assert.equal(subjectOfConditionKey("rule:1:server:1"), "server:1");
|
|
assert.equal(subjectOfConditionKey("rule:1:server:4:/data"), "server:4");
|
|
assert.deepEqual(metaOfConditionKey("rule:1:server:4", "critical"), { category: "monitoring", severity: "critical", serverId: 4 });
|
|
const down = evaluateHealth(snap(new Date(NOW - 3 * 3600_000).toISOString()), thresholds, NOW).map((c) => c.key);
|
|
assert.deepEqual(down, ["offline:server:1"], "an offline server says so, and is not also judged on its load");
|
|
assert.deepEqual(evaluateHealth(snap(new Date(NOW - MIN).toISOString()), thresholds, NOW, { skipServers: true }), []);
|
|
});
|
|
|
|
test("what is asked for is checked, and said plainly when it is wrong", async () => {
|
|
const bad = async (input: Parameters<typeof checkedRule>[0]) => checkedRule(input).then(() => null, (e) => (e instanceof RuleError ? e.code : "other"));
|
|
assert.equal(await bad({ name: " ", metric: "cpu", threshold: 90 }), "invalid_name");
|
|
assert.equal(await bad({ name: "x", metric: "cpu", threshold: 120 }), "invalid_threshold");
|
|
assert.equal(await bad({ name: "x", metric: "disk_temp", threshold: 200 }), "invalid_threshold");
|
|
assert.equal(await bad({ name: "x", metric: "cpu", threshold: 90, forMinutes: 5000 }), "invalid_for");
|
|
assert.equal(await bad({ name: "x", metric: "cpu", threshold: 90, tag: "a b c d e f g h i j k l m n o p q r s t u v w x y z" }), "invalid_tag");
|
|
assert.equal(await bad({ name: "x", metric: "cpu", threshold: 90, serverId: 999, }), "unknown_server");
|
|
const ok = await checkedRule({ name: " Tidy ", metric: "disk_temp", threshold: 55, forMinutes: 30, mount: "/x", tag: " Prod " });
|
|
assert.deepEqual([ok.name, ok.forMinutes, ok.mount, ok.tag], ["Tidy", 0, null, "prod"], "a temperature has no time and no mount; a tag is written the usual way");
|
|
});
|
|
});
|
|
|
|
describe("rules, the Alerts page and the pages that show them", () => {
|
|
let app: Server;
|
|
let base = "";
|
|
let who = "viewer";
|
|
let webId = 0;
|
|
|
|
before(async () => {
|
|
await db.insert(users).values([
|
|
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
|
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
|
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
|
]);
|
|
const now = new Date();
|
|
const [web] = await db.insert(servers).values({ name: "web", apiTokenHash: "h1", apiTokenPrefix: "p1", lastSeenAt: now.toISOString(), tags: JSON.stringify(["prod"]), ipAddresses: "[]" }).returning();
|
|
webId = web.id;
|
|
await db.insert(servers).values({ name: "pi", apiTokenHash: "h2", apiTokenPrefix: "p2", lastSeenAt: now.toISOString(), tags: JSON.stringify(["lab"]), ipAddresses: "[]" });
|
|
// web has been at 95% for thirty minutes; pi is fine.
|
|
for (const [id, vals] of [[web.id, [95, 96, 97]], [web.id + 1, [20, 25, 22]]] as const) {
|
|
for (let i = 0; i < vals.length; i++) await db.insert(serverMetrics).values({ serverId: id, metric: "cpu", value: vals[i], recordedAt: new Date(now.getTime() - (vals.length - 1 - i) * 15 * MIN - MIN).toISOString() });
|
|
}
|
|
const a = express();
|
|
a.use(express.json());
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
|
next();
|
|
});
|
|
a.use("/api/alert-rules", alertRulesRouter);
|
|
a.use("/api/alerts", alertsRouter);
|
|
a.use("/api/storage", storageRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
|
});
|
|
after(() => app.close());
|
|
|
|
const send = async (method: string, path: string, body?: unknown) => {
|
|
const res = await fetch(`${base}${path}`, { method, headers: { "content-type": "application/json", connection: "close" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
|
return { status: res.status, body: res.status === 204 ? null : ((await res.json()) as any) };
|
|
};
|
|
const draft = { name: "Web CPU", metric: "cpu", threshold: 90, forMinutes: 15, severity: "critical" };
|
|
|
|
test("everyone reads the rules; only an administrator makes, tries, changes and removes them", async () => {
|
|
who = "operator";
|
|
assert.equal((await send("POST", "/alert-rules", draft)).status, 403);
|
|
assert.equal((await send("POST", "/alert-rules/preview", draft)).status, 403);
|
|
who = "admin";
|
|
assert.equal((await send("POST", "/alert-rules", { ...draft, threshold: 500 })).body.error, "invalid_threshold");
|
|
const preview = await send("POST", "/alert-rules/preview", draft);
|
|
assert.deepEqual(preview.body.findings.map((f: any) => f.serverName), ["web"], "tried before it is saved");
|
|
assert.equal((await db.select().from(alertRules)).length, 0, "and saving nothing");
|
|
const made = await send("POST", "/alert-rules", draft);
|
|
assert.equal(made.status, 201);
|
|
who = "viewer";
|
|
const list = await send("GET", "/alert-rules");
|
|
assert.equal(list.status, 200);
|
|
assert.deepEqual(list.body.rules.map((r: any) => [r.name, r.firing.map((f: any) => f.serverName)]), [["Web CPU", ["web"]]]);
|
|
assert.deepEqual(list.body.tags, ["lab", "prod"]);
|
|
assert.equal((await send("DELETE", `/alert-rules/${made.body.rule.id}`)).status, 403);
|
|
});
|
|
|
|
test("a rule is a monitoring alert about the server, with its own severity, where to look, and who to silence it with", async () => {
|
|
invalidateAlertsCache();
|
|
const report = await getAlerts(true);
|
|
const rid = (await listRules())[0].id;
|
|
const alert = report.alerts.find((a) => a.id === `monitoring:rule:${rid}:server:${webId}`)!;
|
|
assert.ok(alert, "raised");
|
|
assert.deepEqual([alert.severity, alert.category, alert.source, alert.link, alert.target?.name, alert.silenced], ["critical", "monitoring", "Server", `/servers/${webId}?view=history`, "web", false]);
|
|
assert.match(alert.message, /web: cpu load is 97%, above 90% for 15 min \[Web CPU\]/);
|
|
assert.equal(report.alerts.some((a) => a.id.includes("server:" + (webId + 1)) && a.id.startsWith("monitoring:rule")), false, "pi is fine");
|
|
const [win] = await db.insert(maintenanceWindows).values({ targetType: "server", targetId: webId, startedAt: new Date(Date.now() - 1000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() }).returning();
|
|
invalidateAlertsCache();
|
|
assert.equal((await getAlerts(true)).alerts.find((a) => a.id === alert.id)!.silenced, true, "maintenance holds it");
|
|
await db.delete(maintenanceWindows).where(eq(maintenanceWindows.id, win.id));
|
|
});
|
|
|
|
test("switching it off, or changing the limit, takes the alert away; removing it too", async () => {
|
|
who = "admin";
|
|
const r = (await listRules())[0];
|
|
assert.equal((await send("PUT", `/alert-rules/${r.id}`, { ...draft, enabled: false })).status, 200);
|
|
invalidateAlertsCache();
|
|
assert.equal((await getAlerts(true)).alerts.some((a) => a.id.startsWith("monitoring:rule:")), false);
|
|
assert.equal((await send("PUT", `/alert-rules/${r.id}`, { ...draft, threshold: 99 })).status, 200);
|
|
assert.deepEqual((await computeRuleFindings(await listRules())), [], "99 is above all it has seen");
|
|
assert.equal((await send("PUT", "/alert-rules/99999", draft)).status, 404);
|
|
assert.equal((await send("DELETE", `/alert-rules/${r.id}`)).status, 204);
|
|
assert.equal((await send("DELETE", `/alert-rules/${r.id}`)).status, 404);
|
|
});
|
|
|
|
describe("storage forecasts", () => {
|
|
const px = (usedGb: number, extra: { shared?: boolean; node?: string } = {}) => ({
|
|
proxmox: [{ integrationId: 1, integrationName: "PVE", nodes: [{ node: extra.node ?? "pve1", error: null, uptime: 1, cpuUsagePercent: 1, cpuCores: 4, loadAverage: null, memTotalBytes: 1, memUsedBytes: 1, swapTotalBytes: null, swapUsedBytes: null, rootfsTotalBytes: 100e9, rootfsUsedBytes: 20e9, pveVersion: null, storages: [{ id: "tank", type: "zfs", active: true, shared: extra.shared ?? false, totalBytes: 1000e9, usedBytes: usedGb * 1e9, availBytes: null }, { id: "off", type: "dir", active: false, shared: false, totalBytes: 10, usedBytes: 5, availBytes: null }] }, { node: "pve2", error: "no access", uptime: null, cpuUsagePercent: null, cpuCores: null, loadAverage: null, memTotalBytes: null, memUsedBytes: null, swapTotalBytes: null, swapUsedBytes: null, rootfsTotalBytes: null, rootfsUsedBytes: null, pveVersion: null, storages: [] }] }],
|
|
synology: [{ integrationId: 2, integrationName: "NAS", storage: { volumes: [{ id: "volume_1", status: "normal", deviceType: "raid", sizeTotal: 8e12, sizeUsed: 6e12 }, { id: "volume_2", status: "normal", deviceType: "raid", sizeTotal: null, sizeUsed: null }], disks: [] } }],
|
|
});
|
|
|
|
test("the pools in a read: root filesystems, active storages (a shared one once), volumes; not a node that could not be read, an inactive storage or a volume with no size", () => {
|
|
const pools = poolsOf(px(500));
|
|
assert.deepEqual(pools.map((p) => [p.integrationId, p.key, Math.round(p.percent)]), [[1, "pve1:rootfs", 20], [1, "pve1:storage:tank", 50], [2, "volume:volume_1", 75]]);
|
|
const shared = poolsOf({ ...px(500, { shared: true }), proxmox: [{ ...px(500, { shared: true }).proxmox[0], nodes: [px(500, { shared: true }).proxmox[0].nodes[0], { ...px(500, { shared: true }).proxmox[0].nodes[0], node: "pve3" }] }] });
|
|
assert.equal(shared.filter((p) => p.key === "shared:tank").length, 1, "listed by every node, said once");
|
|
});
|
|
|
|
test("a reading is kept an hour at most; the pool's latest figure is always the newest; a pool that is not reported for a month is forgotten", async () => {
|
|
await db.insert(integrations).values([{ type: "proxmox", name: "PVE", baseUrl: "http://pve" }, { type: "synology", name: "NAS", baseUrl: "http://nas" }]);
|
|
const t0 = new Date(NOW);
|
|
assert.equal(await recordStorageSamples(px(500), t0), 3);
|
|
assert.equal(await recordStorageSamples(px(505), new Date(NOW + 20 * MIN)), 0, "too soon");
|
|
assert.equal(await recordStorageSamples(px(510), new Date(NOW + 70 * MIN)), 3);
|
|
const series = await db.select().from(storageSeries);
|
|
assert.equal(series.length, 3);
|
|
assert.equal(series.find((s) => s.key === "pve1:storage:tank")!.latestPercent, 51, "the latest, from the reading that was kept");
|
|
assert.equal((await db.select().from(storageSamples)).length, 6);
|
|
await recordStorageSamples({ proxmox: [], synology: [] }, new Date(NOW + 40 * DAY));
|
|
assert.equal((await db.select().from(storageSeries)).length, 0, "forgotten with its readings");
|
|
assert.equal((await db.select().from(storageSamples)).length, 0);
|
|
});
|
|
|
|
test("a pool on a steady climb is forecast; a flat one, a wobbly one and one already over the limit are not", async () => {
|
|
// This one is judged on the Alerts page and by the routes, which read the real clock (a pool not reported for two days is not believed), so its history is laid out from the real now.
|
|
const T = Date.now();
|
|
const [pve] = await db.select().from(integrations).where(eq(integrations.type, "proxmox"));
|
|
const make = async (key: string, label: string, valueAt: (day: number) => number, latest: number) => {
|
|
const [s] = await db.insert(storageSeries).values({ integrationId: pve.id, key, label, latestPercent: latest, usedBytes: null, totalBytes: null, updatedAt: new Date(T).toISOString() }).returning();
|
|
for (let d = 0; d <= 14; d += 0.5) await db.insert(storageSamples).values({ seriesId: s.id, at: new Date(T - (14 - d) * DAY).toISOString(), value: valueAt(d) });
|
|
return s;
|
|
};
|
|
const climbing = await make("a:climb", "PVE / a: storage \"climb\"", (d) => 50 + d * 2.5, 85);
|
|
await make("b:flat", "PVE / b: storage \"flat\"", () => 40, 40);
|
|
await make("c:wobble", "PVE / c: storage \"wobble\"", (d) => 50 + (Math.floor(d * 2) % 2 === 0 ? 8 : -8), 58);
|
|
await make("d:over", "PVE / d: storage \"over\"", (d) => 80 + d * 0.9, 92.6);
|
|
const rows = await storageOverview(new Date(T));
|
|
const by = Object.fromEntries(rows.map((r) => [r.key, r]));
|
|
assert.equal(by["a:climb"].steady, true);
|
|
assert.equal(by["b:flat"].trend?.daysToFull ?? null, null);
|
|
assert.equal(by["c:wobble"].steady, false);
|
|
const soon = storageFillingSoon(rows, 30, 90);
|
|
assert.deepEqual(soon.map((f) => f.key), ["a:climb"], "over the limit, flat and erratic are left out");
|
|
assert.equal(soon[0].daysToFull, Math.ceil((100 - 85) / 2.5));
|
|
assert.equal(storageFillingSoon(rows, 5, 90).length, 0, "not within five days");
|
|
|
|
// On the Alerts page: raised, silenced with the integration, and gone when the forecast is turned off.
|
|
invalidateAlertsCache();
|
|
const alert = (await getAlerts(true)).alerts.find((a) => a.id === `disk:storage-forecast:${pve.id}:a:climb`)!;
|
|
assert.ok(alert);
|
|
assert.deepEqual([alert.severity, alert.source, alert.target?.type, alert.silenced], ["warning", "Proxmox", "integration", false]);
|
|
assert.match(alert.message, /filling by about 2\.5 points a day — full in about 6 days \(85% used now\)/);
|
|
who = "viewer";
|
|
const api = await send("GET", "/storage");
|
|
assert.equal(api.status, 200);
|
|
assert.equal(api.body.pools.find((p: any) => p.key === "a:climb").steady, true);
|
|
const hist = await send("GET", `/storage/${climbing.id}/history?days=30`);
|
|
assert.ok(hist.body.points.length > 5);
|
|
});
|
|
});
|
|
|
|
describe("snooze", () => {
|
|
const inHours = (h: number) => new Date(Date.now() + h * 3600_000).toISOString();
|
|
const alertIdOf = async (prefix: string) => (await getAlerts(true)).alerts.find((a) => a.id.startsWith(prefix))!;
|
|
|
|
test("only an operator snoozes; a time that has passed, or is a year off, or an alert that is not there, is refused", async () => {
|
|
invalidateAlertsCache();
|
|
const target = await alertIdOf("disk:storage-forecast");
|
|
assert.ok(target, "something to snooze");
|
|
who = "viewer";
|
|
assert.equal((await send("PUT", "/alerts/snooze", { alertId: target.id, until: inHours(5) })).status, 403);
|
|
who = "operator";
|
|
assert.equal((await send("PUT", "/alerts/snooze", { alertId: target.id, until: inHours(-1) })).body.error, "invalid_until");
|
|
assert.equal((await send("PUT", "/alerts/snooze", { alertId: target.id, until: inHours(24 * 400) })).body.error, "invalid_until");
|
|
assert.equal((await send("PUT", "/alerts/snooze", { alertId: target.id, until: "tomorrow" })).status, 400);
|
|
assert.equal((await send("PUT", "/alerts/snooze", { alertId: "disk:nothing", until: inHours(5) })).status, 404);
|
|
});
|
|
|
|
test("a snoozed alert is held back like one under maintenance, with when it returns; waking it brings it back at once", async () => {
|
|
who = "operator";
|
|
const target = await alertIdOf("disk:storage-forecast");
|
|
const until = inHours(18);
|
|
const res = await send("PUT", "/alerts/snooze", { alertId: target.id, until });
|
|
assert.equal(res.status, 200);
|
|
const page = await send("GET", "/alerts");
|
|
const row = page.body.alerts.find((a: any) => a.id === target.id);
|
|
assert.deepEqual([row.silenced, row.snoozed.until, row.snoozed.by], [true, new Date(until).toISOString(), "Olga"]);
|
|
assert.ok(page.body.counts.silenced >= 1, "counted with the held ones, not with those that want a look");
|
|
assert.equal((await send("DELETE", `/alerts/snooze?alertId=${encodeURIComponent(target.id)}`)).status, 204);
|
|
assert.equal((await send("DELETE", `/alerts/snooze?alertId=${encodeURIComponent(target.id)}`)).status, 404);
|
|
const again = (await send("GET", "/alerts")).body.alerts.find((a: any) => a.id === target.id);
|
|
assert.deepEqual([again.silenced, again.snoozed ?? null], [false, null]);
|
|
});
|
|
|
|
test("messages about a snoozed alert are held; those about others are not; when the time is up a reminder is made for one that is still there, and nothing for one that went", async () => {
|
|
const now = new Date();
|
|
await setSnooze({ alertId: "disk:disk:server:1:/", conditionKey: "disk:server:1:/", message: "web: / is 95% full", until: new Date(now.getTime() + 2 * 3600_000), actor: "Olga" }, now);
|
|
assert.deepEqual((await withoutSnoozed([{ key: "disk:server:1:/", message: "a" }, { key: "disk:server:2:/", message: "b" }, { message: "no key" }], now)).map((i) => i.message), ["b", "no key"]);
|
|
assert.equal((await loadSnoozes(now)).has("disk:disk:server:1:/"), true);
|
|
await assert.rejects(setSnooze({ alertId: "x", conditionKey: "x", message: "x", until: new Date(now.getTime() + 10_000), actor: "Olga" }, now), SnoozeError);
|
|
|
|
// Two hours and a bit later: it is still there, so it is a reminder; the snooze is gone, so the next message about it is heard.
|
|
const later = new Date(now.getTime() + 2 * 3600_000 + MIN);
|
|
await setSnooze({ alertId: "backup:gone", conditionKey: "gone", message: "gone soon", until: new Date(now.getTime() + 3 * 3600_000), actor: "Olga" }, now);
|
|
const reminders = await settleSnoozes([{ id: "disk:disk:server:1:/", message: "web: / is 96% full", severity: "warning", key: "disk:server:1:/" }], later);
|
|
assert.deepEqual(reminders, [{ key: "disk:server:1:/", message: "web: / is 96% full", severity: "warning" }]);
|
|
assert.equal((await withoutSnoozed([{ key: "disk:server:1:/", message: "a" }], later)).length, 1);
|
|
// One whose alert is gone, and has been for half an hour, is dropped quietly; the snooze ending for an alert that went is forgotten without a word.
|
|
const muchLater = new Date(now.getTime() + 4 * 3600_000);
|
|
assert.deepEqual(await settleSnoozes([], muchLater), []);
|
|
assert.equal((await db.select().from(alertSnoozes)).length, 0);
|
|
assert.equal(await clearSnooze("disk:disk:server:1:/"), null);
|
|
});
|
|
});
|
|
});
|