Alert rules of your own, storage forecasts for Proxmox and Synology, and snooze
- Monitoring -> Alert rules: CPU, memory, disk use and disk temperature limits with a time, severity and scope (all, one server, a tag); try before saving; findings are ordinary health conditions - Proxmox storages and Synology volumes are written down on each health pass (hourly, 400 days) and forecast like server disks: badge and history chart on the pages, alert, digest line, /metrics gauge - Snooze an alert until a time; held back like maintenance, messages held, announced once again if still there when it ends - Docs updated Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
788b3e073c
commit
24ae8db380
38 files changed
+7847
-8
No files matched your search
@@ -0,0 +1,325 @@
|
||||
/**
|
||||
* Alert rules of your own, forecasts for the pools the integrations report, and snoozing an alert.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-rss-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, serverMetrics, integrations, alertRules, storageSeries, storageSamples, alertSnoozes, maintenanceWindows } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { evaluateRules, currentRun, checkedRule, computeRuleFindings, listRules, findingKey, ruleCovers, RuleError } = await import("../src/services/alertRules.js");
|
||||
const { evaluateHealth } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { metaOfConditionKey } = await import("../src/services/notifyRouting.js");
|
||||
const { poolsOf, recordStorageSamples, storageOverview, storageFillingSoon } = await import("../src/services/storageHistory.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { setSnooze, clearSnooze, loadSnoozes, withoutSnoozed, settleSnoozes, SnoozeError } = await import("../src/services/alertSnoozes.js");
|
||||
const { alertRulesRouter } = await import("../src/routes/alertRules.js");
|
||||
const { alertsRouter } = await import("../src/routes/alerts.js");
|
||||
const { storageRouter } = await import("../src/routes/storage.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const DAY = 86_400_000;
|
||||
const NOW = Date.parse("2026-10-20T12:00:00Z");
|
||||
type Rule = Parameters<typeof evaluateRules>[0][number];
|
||||
const rule = (extra: Partial<Rule> = {}): Rule => ({ id: 1, name: "CPU hot", enabled: true, metric: "cpu", mount: null, operator: "above", threshold: 90, forMinutes: 15, severity: "warning", serverId: null, tag: null, createdBy: null, createdAt: "", ...extra });
|
||||
const samplesAt = (values: number[], endsAgoMin = 1, gapMin = 15) => values.map((v, i) => ({ t: NOW - (endsAgoMin + (values.length - 1 - i) * gapMin) * MIN, v }));
|
||||
const server = (id: number, name: string, tags: string[] = [], smart: { device: string; temperatureC: number | null }[] = []) => ({ id, name, tags, smart });
|
||||
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows may still hold the database file
|
||||
}
|
||||
});
|
||||
|
||||
describe("your own alert rules", () => {
|
||||
test("it has to have been past the limit for the whole time, on every sample, up to now", () => {
|
||||
assert.equal(currentRun(rule(), samplesAt([95]), NOW)?.since, NOW - 1 * MIN);
|
||||
const held = currentRun(rule(), samplesAt([95, 96, 97]), NOW)!;
|
||||
assert.deepEqual([held.value, (held.latest - held.since) / MIN], [97, 30]);
|
||||
assert.equal(currentRun(rule(), samplesAt([95, 60, 97]), NOW)!.since, NOW - 1 * MIN, "a dip starts the count again");
|
||||
assert.equal(currentRun(rule(), samplesAt([95, 96, 60]), NOW), null, "the newest sample is fine");
|
||||
assert.equal(currentRun(rule(), samplesAt([95, 96, 97], 50), NOW), null, "too old to say what it is now");
|
||||
assert.equal(currentRun(rule({ operator: "below", threshold: 10 }), samplesAt([5, 4]), NOW)!.value, 4);
|
||||
});
|
||||
|
||||
test("a rule with a time waits for it; without one it speaks at once; a different server or tag is left alone", () => {
|
||||
const s = [server(1, "web", ["prod"]), server(2, "pi", ["lab"])];
|
||||
const hot = new Map([["1|cpu", samplesAt([95, 96])], ["2|cpu", samplesAt([95, 96])]]);
|
||||
const found = (r: Rule, m = hot) => evaluateRules([r], s, m, NOW).map((f) => f.serverName);
|
||||
assert.deepEqual(found(rule()), ["web", "pi"], "30 minutes past the limit is more than the 15 asked");
|
||||
assert.deepEqual(found(rule({ forMinutes: 60 })), [], "but not an hour");
|
||||
assert.deepEqual(found(rule({ forMinutes: 0 }), new Map([["1|cpu", samplesAt([95])]])), ["web"], "one sample is enough without a time");
|
||||
assert.deepEqual(found(rule({ tag: "LAB" })), ["pi"], "a tag, however it is written");
|
||||
assert.deepEqual(found(rule({ serverId: 1 })), ["web"]);
|
||||
assert.equal(ruleCovers({ serverId: null, tag: null }, { id: 9, tags: [] }), true);
|
||||
assert.deepEqual(evaluateRules([rule({ enabled: false })], s, hot, NOW), []);
|
||||
});
|
||||
|
||||
test("a disk is judged mount by mount, or just the one named; a temperature is judged as of the last report, per disk", () => {
|
||||
const s = [server(1, "nas", [], [{ device: "/dev/sda", temperatureC: 58 }, { device: "/dev/sdb", temperatureC: 41 }, { device: "/dev/sdc", temperatureC: null }])];
|
||||
const disks = new Map([["1|disk:/", samplesAt([88, 92])], ["1|disk:/data", samplesAt([91, 93])], ["1|cpu", samplesAt([99])]]);
|
||||
const every = evaluateRules([rule({ metric: "disk", threshold: 90, forMinutes: 0 })], s, disks, NOW);
|
||||
assert.deepEqual(every.map((f) => f.detail), ["/", "/data"]);
|
||||
assert.match(every[1].message, /nas: \/data use is 93%, above 90% \[CPU hot\]/);
|
||||
assert.deepEqual(evaluateRules([rule({ metric: "disk", threshold: 90, forMinutes: 0, mount: "/data" })], s, disks, NOW).map((f) => f.detail), ["/data"]);
|
||||
const temp = evaluateRules([rule({ metric: "disk_temp", threshold: 55, forMinutes: 0, severity: "critical", name: "Warm disk" })], s, new Map(), NOW);
|
||||
assert.deepEqual(temp.map((f) => [f.detail, f.value, f.severity, f.since]), [["/dev/sda", 58, "critical", null]]);
|
||||
assert.match(temp[0].message, /nas: \/dev\/sda is 58 °C, above 55 °C \[Warm disk\]/);
|
||||
});
|
||||
|
||||
test("a finding is a health condition of the server — announced, silenced by maintenance and routed like the rest — unless the server is offline", () => {
|
||||
const f = evaluateRules([rule()], [server(1, "web")], new Map([["1|cpu", samplesAt([95, 96])]]), NOW);
|
||||
assert.equal(findingKey(f[0]), "rule:1:server:1");
|
||||
assert.equal(findingKey({ ruleId: 2, serverId: 4, detail: "/data" }), "rule:2:server:4:/data");
|
||||
const snap = (lastSeenAt: string) => ({ servers: [{ id: 1, name: "web", lastSeenAt, disks: [] }], proxmox: [], synology: [], rules: f });
|
||||
const thresholds = { serverOfflineMinutes: 60, diskUsagePercent: 90 };
|
||||
const ok = evaluateHealth(snap(new Date(NOW - MIN).toISOString()), thresholds, NOW);
|
||||
assert.deepEqual(ok.map((c) => [c.key, c.source, c.severity]), [["rule:1:server:1", "server:rules", "warning"]]);
|
||||
assert.equal(subjectOfConditionKey("rule:1:server:1"), "server:1");
|
||||
assert.equal(subjectOfConditionKey("rule:1:server:4:/data"), "server:4");
|
||||
assert.deepEqual(metaOfConditionKey("rule:1:server:4", "critical"), { category: "monitoring", severity: "critical", serverId: 4 });
|
||||
const down = evaluateHealth(snap(new Date(NOW - 3 * 3600_000).toISOString()), thresholds, NOW).map((c) => c.key);
|
||||
assert.deepEqual(down, ["offline:server:1"], "an offline server says so, and is not also judged on its load");
|
||||
assert.deepEqual(evaluateHealth(snap(new Date(NOW - MIN).toISOString()), thresholds, NOW, { skipServers: true }), []);
|
||||
});
|
||||
|
||||
test("what is asked for is checked, and said plainly when it is wrong", async () => {
|
||||
const bad = async (input: Parameters<typeof checkedRule>[0]) => checkedRule(input).then(() => null, (e) => (e instanceof RuleError ? e.code : "other"));
|
||||
assert.equal(await bad({ name: " ", metric: "cpu", threshold: 90 }), "invalid_name");
|
||||
assert.equal(await bad({ name: "x", metric: "cpu", threshold: 120 }), "invalid_threshold");
|
||||
assert.equal(await bad({ name: "x", metric: "disk_temp", threshold: 200 }), "invalid_threshold");
|
||||
assert.equal(await bad({ name: "x", metric: "cpu", threshold: 90, forMinutes: 5000 }), "invalid_for");
|
||||
assert.equal(await bad({ name: "x", metric: "cpu", threshold: 90, tag: "a b c d e f g h i j k l m n o p q r s t u v w x y z" }), "invalid_tag");
|
||||
assert.equal(await bad({ name: "x", metric: "cpu", threshold: 90, serverId: 999, }), "unknown_server");
|
||||
const ok = await checkedRule({ name: " Tidy ", metric: "disk_temp", threshold: 55, forMinutes: 30, mount: "/x", tag: " Prod " });
|
||||
assert.deepEqual([ok.name, ok.forMinutes, ok.mount, ok.tag], ["Tidy", 0, null, "prod"], "a temperature has no time and no mount; a tag is written the usual way");
|
||||
});
|
||||
});
|
||||
|
||||
describe("rules, the Alerts page and the pages that show them", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "viewer";
|
||||
let webId = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
const now = new Date();
|
||||
const [web] = await db.insert(servers).values({ name: "web", apiTokenHash: "h1", apiTokenPrefix: "p1", lastSeenAt: now.toISOString(), tags: JSON.stringify(["prod"]), ipAddresses: "[]" }).returning();
|
||||
webId = web.id;
|
||||
await db.insert(servers).values({ name: "pi", apiTokenHash: "h2", apiTokenPrefix: "p2", lastSeenAt: now.toISOString(), tags: JSON.stringify(["lab"]), ipAddresses: "[]" });
|
||||
// web has been at 95% for thirty minutes; pi is fine.
|
||||
for (const [id, vals] of [[web.id, [95, 96, 97]], [web.id + 1, [20, 25, 22]]] as const) {
|
||||
for (let i = 0; i < vals.length; i++) await db.insert(serverMetrics).values({ serverId: id, metric: "cpu", value: vals[i], recordedAt: new Date(now.getTime() - (vals.length - 1 - i) * 15 * MIN - MIN).toISOString() });
|
||||
}
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/alert-rules", alertRulesRouter);
|
||||
a.use("/api/alerts", alertsRouter);
|
||||
a.use("/api/storage", storageRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
const send = async (method: string, path: string, body?: unknown) => {
|
||||
const res = await fetch(`${base}${path}`, { method, headers: { "content-type": "application/json", connection: "close" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
return { status: res.status, body: res.status === 204 ? null : ((await res.json()) as any) };
|
||||
};
|
||||
const draft = { name: "Web CPU", metric: "cpu", threshold: 90, forMinutes: 15, severity: "critical" };
|
||||
|
||||
test("everyone reads the rules; only an administrator makes, tries, changes and removes them", async () => {
|
||||
who = "operator";
|
||||
assert.equal((await send("POST", "/alert-rules", draft)).status, 403);
|
||||
assert.equal((await send("POST", "/alert-rules/preview", draft)).status, 403);
|
||||
who = "admin";
|
||||
assert.equal((await send("POST", "/alert-rules", { ...draft, threshold: 500 })).body.error, "invalid_threshold");
|
||||
const preview = await send("POST", "/alert-rules/preview", draft);
|
||||
assert.deepEqual(preview.body.findings.map((f: any) => f.serverName), ["web"], "tried before it is saved");
|
||||
assert.equal((await db.select().from(alertRules)).length, 0, "and saving nothing");
|
||||
const made = await send("POST", "/alert-rules", draft);
|
||||
assert.equal(made.status, 201);
|
||||
who = "viewer";
|
||||
const list = await send("GET", "/alert-rules");
|
||||
assert.equal(list.status, 200);
|
||||
assert.deepEqual(list.body.rules.map((r: any) => [r.name, r.firing.map((f: any) => f.serverName)]), [["Web CPU", ["web"]]]);
|
||||
assert.deepEqual(list.body.tags, ["lab", "prod"]);
|
||||
assert.equal((await send("DELETE", `/alert-rules/${made.body.rule.id}`)).status, 403);
|
||||
});
|
||||
|
||||
test("a rule is a monitoring alert about the server, with its own severity, where to look, and who to silence it with", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const rid = (await listRules())[0].id;
|
||||
const alert = report.alerts.find((a) => a.id === `monitoring:rule:${rid}:server:${webId}`)!;
|
||||
assert.ok(alert, "raised");
|
||||
assert.deepEqual([alert.severity, alert.category, alert.source, alert.link, alert.target?.name, alert.silenced], ["critical", "monitoring", "Server", `/servers/${webId}?view=history`, "web", false]);
|
||||
assert.match(alert.message, /web: cpu load is 97%, above 90% for 15 min \[Web CPU\]/);
|
||||
assert.equal(report.alerts.some((a) => a.id.includes("server:" + (webId + 1)) && a.id.startsWith("monitoring:rule")), false, "pi is fine");
|
||||
const [win] = await db.insert(maintenanceWindows).values({ targetType: "server", targetId: webId, startedAt: new Date(Date.now() - 1000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() }).returning();
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await getAlerts(true)).alerts.find((a) => a.id === alert.id)!.silenced, true, "maintenance holds it");
|
||||
await db.delete(maintenanceWindows).where(eq(maintenanceWindows.id, win.id));
|
||||
});
|
||||
|
||||
test("switching it off, or changing the limit, takes the alert away; removing it too", async () => {
|
||||
who = "admin";
|
||||
const r = (await listRules())[0];
|
||||
assert.equal((await send("PUT", `/alert-rules/${r.id}`, { ...draft, enabled: false })).status, 200);
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await getAlerts(true)).alerts.some((a) => a.id.startsWith("monitoring:rule:")), false);
|
||||
assert.equal((await send("PUT", `/alert-rules/${r.id}`, { ...draft, threshold: 99 })).status, 200);
|
||||
assert.deepEqual((await computeRuleFindings(await listRules())), [], "99 is above all it has seen");
|
||||
assert.equal((await send("PUT", "/alert-rules/99999", draft)).status, 404);
|
||||
assert.equal((await send("DELETE", `/alert-rules/${r.id}`)).status, 204);
|
||||
assert.equal((await send("DELETE", `/alert-rules/${r.id}`)).status, 404);
|
||||
});
|
||||
|
||||
describe("storage forecasts", () => {
|
||||
const px = (usedGb: number, extra: { shared?: boolean; node?: string } = {}) => ({
|
||||
proxmox: [{ integrationId: 1, integrationName: "PVE", nodes: [{ node: extra.node ?? "pve1", error: null, uptime: 1, cpuUsagePercent: 1, cpuCores: 4, loadAverage: null, memTotalBytes: 1, memUsedBytes: 1, swapTotalBytes: null, swapUsedBytes: null, rootfsTotalBytes: 100e9, rootfsUsedBytes: 20e9, pveVersion: null, storages: [{ id: "tank", type: "zfs", active: true, shared: extra.shared ?? false, totalBytes: 1000e9, usedBytes: usedGb * 1e9, availBytes: null }, { id: "off", type: "dir", active: false, shared: false, totalBytes: 10, usedBytes: 5, availBytes: null }] }, { node: "pve2", error: "no access", uptime: null, cpuUsagePercent: null, cpuCores: null, loadAverage: null, memTotalBytes: null, memUsedBytes: null, swapTotalBytes: null, swapUsedBytes: null, rootfsTotalBytes: null, rootfsUsedBytes: null, pveVersion: null, storages: [] }] }],
|
||||
synology: [{ integrationId: 2, integrationName: "NAS", storage: { volumes: [{ id: "volume_1", status: "normal", deviceType: "raid", sizeTotal: 8e12, sizeUsed: 6e12 }, { id: "volume_2", status: "normal", deviceType: "raid", sizeTotal: null, sizeUsed: null }], disks: [] } }],
|
||||
});
|
||||
|
||||
test("the pools in a read: root filesystems, active storages (a shared one once), volumes; not a node that could not be read, an inactive storage or a volume with no size", () => {
|
||||
const pools = poolsOf(px(500));
|
||||
assert.deepEqual(pools.map((p) => [p.integrationId, p.key, Math.round(p.percent)]), [[1, "pve1:rootfs", 20], [1, "pve1:storage:tank", 50], [2, "volume:volume_1", 75]]);
|
||||
const shared = poolsOf({ ...px(500, { shared: true }), proxmox: [{ ...px(500, { shared: true }).proxmox[0], nodes: [px(500, { shared: true }).proxmox[0].nodes[0], { ...px(500, { shared: true }).proxmox[0].nodes[0], node: "pve3" }] }] });
|
||||
assert.equal(shared.filter((p) => p.key === "shared:tank").length, 1, "listed by every node, said once");
|
||||
});
|
||||
|
||||
test("a reading is kept an hour at most; the pool's latest figure is always the newest; a pool that is not reported for a month is forgotten", async () => {
|
||||
await db.insert(integrations).values([{ type: "proxmox", name: "PVE", baseUrl: "http://pve" }, { type: "synology", name: "NAS", baseUrl: "http://nas" }]);
|
||||
const t0 = new Date(NOW);
|
||||
assert.equal(await recordStorageSamples(px(500), t0), 3);
|
||||
assert.equal(await recordStorageSamples(px(505), new Date(NOW + 20 * MIN)), 0, "too soon");
|
||||
assert.equal(await recordStorageSamples(px(510), new Date(NOW + 70 * MIN)), 3);
|
||||
const series = await db.select().from(storageSeries);
|
||||
assert.equal(series.length, 3);
|
||||
assert.equal(series.find((s) => s.key === "pve1:storage:tank")!.latestPercent, 51, "the latest, from the reading that was kept");
|
||||
assert.equal((await db.select().from(storageSamples)).length, 6);
|
||||
await recordStorageSamples({ proxmox: [], synology: [] }, new Date(NOW + 40 * DAY));
|
||||
assert.equal((await db.select().from(storageSeries)).length, 0, "forgotten with its readings");
|
||||
assert.equal((await db.select().from(storageSamples)).length, 0);
|
||||
});
|
||||
|
||||
test("a pool on a steady climb is forecast; a flat one, a wobbly one and one already over the limit are not", async () => {
|
||||
const [pve] = await db.select().from(integrations).where(eq(integrations.type, "proxmox"));
|
||||
const make = async (key: string, label: string, valueAt: (day: number) => number, latest: number) => {
|
||||
const [s] = await db.insert(storageSeries).values({ integrationId: pve.id, key, label, latestPercent: latest, usedBytes: null, totalBytes: null, updatedAt: new Date(NOW).toISOString() }).returning();
|
||||
for (let d = 0; d <= 14; d += 0.5) await db.insert(storageSamples).values({ seriesId: s.id, at: new Date(NOW - (14 - d) * DAY).toISOString(), value: valueAt(d) });
|
||||
return s;
|
||||
};
|
||||
const climbing = await make("a:climb", "PVE / a: storage \"climb\"", (d) => 50 + d * 2.5, 85);
|
||||
await make("b:flat", "PVE / b: storage \"flat\"", () => 40, 40);
|
||||
await make("c:wobble", "PVE / c: storage \"wobble\"", (d) => 50 + (Math.floor(d * 2) % 2 === 0 ? 8 : -8), 58);
|
||||
await make("d:over", "PVE / d: storage \"over\"", (d) => 80 + d * 0.9, 92.6);
|
||||
const rows = await storageOverview(new Date(NOW));
|
||||
const by = Object.fromEntries(rows.map((r) => [r.key, r]));
|
||||
assert.equal(by["a:climb"].steady, true);
|
||||
assert.equal(by["b:flat"].trend?.daysToFull ?? null, null);
|
||||
assert.equal(by["c:wobble"].steady, false);
|
||||
const soon = storageFillingSoon(rows, 30, 90);
|
||||
assert.deepEqual(soon.map((f) => f.key), ["a:climb"], "over the limit, flat and erratic are left out");
|
||||
assert.equal(soon[0].daysToFull, Math.ceil((100 - 85) / 2.5));
|
||||
assert.equal(storageFillingSoon(rows, 5, 90).length, 0, "not within five days");
|
||||
|
||||
// On the Alerts page: raised, silenced with the integration, and gone when the forecast is turned off.
|
||||
invalidateAlertsCache();
|
||||
const alert = (await getAlerts(true)).alerts.find((a) => a.id === `disk:storage-forecast:${pve.id}:a:climb`)!;
|
||||
assert.ok(alert);
|
||||
assert.deepEqual([alert.severity, alert.source, alert.target?.type, alert.silenced], ["warning", "Proxmox", "integration", false]);
|
||||
assert.match(alert.message, /filling by about 2\.5 points a day — full in about 6 days \(85% used now\)/);
|
||||
who = "viewer";
|
||||
const api = await send("GET", "/storage");
|
||||
assert.equal(api.status, 200);
|
||||
assert.equal(api.body.pools.find((p: any) => p.key === "a:climb").steady, true);
|
||||
const hist = await send("GET", `/storage/${climbing.id}/history?days=30`);
|
||||
assert.ok(hist.body.points.length > 5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("snooze", () => {
|
||||
const inHours = (h: number) => new Date(Date.now() + h * 3600_000).toISOString();
|
||||
const alertIdOf = async (prefix: string) => (await getAlerts(true)).alerts.find((a) => a.id.startsWith(prefix))!;
|
||||
|
||||
test("only an operator snoozes; a time that has passed, or is a year off, or an alert that is not there, is refused", async () => {
|
||||
invalidateAlertsCache();
|
||||
const target = await alertIdOf("disk:storage-forecast");
|
||||
assert.ok(target, "something to snooze");
|
||||
who = "viewer";
|
||||
assert.equal((await send("PUT", "/alerts/snooze", { alertId: target.id, until: inHours(5) })).status, 403);
|
||||
who = "operator";
|
||||
assert.equal((await send("PUT", "/alerts/snooze", { alertId: target.id, until: inHours(-1) })).body.error, "invalid_until");
|
||||
assert.equal((await send("PUT", "/alerts/snooze", { alertId: target.id, until: inHours(24 * 400) })).body.error, "invalid_until");
|
||||
assert.equal((await send("PUT", "/alerts/snooze", { alertId: target.id, until: "tomorrow" })).status, 400);
|
||||
assert.equal((await send("PUT", "/alerts/snooze", { alertId: "disk:nothing", until: inHours(5) })).status, 404);
|
||||
});
|
||||
|
||||
test("a snoozed alert is held back like one under maintenance, with when it returns; waking it brings it back at once", async () => {
|
||||
who = "operator";
|
||||
const target = await alertIdOf("disk:storage-forecast");
|
||||
const until = inHours(18);
|
||||
const res = await send("PUT", "/alerts/snooze", { alertId: target.id, until });
|
||||
assert.equal(res.status, 200);
|
||||
const page = await send("GET", "/alerts");
|
||||
const row = page.body.alerts.find((a: any) => a.id === target.id);
|
||||
assert.deepEqual([row.silenced, row.snoozed.until, row.snoozed.by], [true, new Date(until).toISOString(), "Olga"]);
|
||||
assert.ok(page.body.counts.silenced >= 1, "counted with the held ones, not with those that want a look");
|
||||
assert.equal((await send("DELETE", `/alerts/snooze?alertId=${encodeURIComponent(target.id)}`)).status, 204);
|
||||
assert.equal((await send("DELETE", `/alerts/snooze?alertId=${encodeURIComponent(target.id)}`)).status, 404);
|
||||
const again = (await send("GET", "/alerts")).body.alerts.find((a: any) => a.id === target.id);
|
||||
assert.deepEqual([again.silenced, again.snoozed ?? null], [false, null]);
|
||||
});
|
||||
|
||||
test("messages about a snoozed alert are held; those about others are not; when the time is up a reminder is made for one that is still there, and nothing for one that went", async () => {
|
||||
const now = new Date();
|
||||
await setSnooze({ alertId: "disk:disk:server:1:/", conditionKey: "disk:server:1:/", message: "web: / is 95% full", until: new Date(now.getTime() + 2 * 3600_000), actor: "Olga" }, now);
|
||||
assert.deepEqual((await withoutSnoozed([{ key: "disk:server:1:/", message: "a" }, { key: "disk:server:2:/", message: "b" }, { message: "no key" }], now)).map((i) => i.message), ["b", "no key"]);
|
||||
assert.equal((await loadSnoozes(now)).has("disk:disk:server:1:/"), true);
|
||||
await assert.rejects(setSnooze({ alertId: "x", conditionKey: "x", message: "x", until: new Date(now.getTime() + 10_000), actor: "Olga" }, now), SnoozeError);
|
||||
|
||||
// Two hours and a bit later: it is still there, so it is a reminder; the snooze is gone, so the next message about it is heard.
|
||||
const later = new Date(now.getTime() + 2 * 3600_000 + MIN);
|
||||
await setSnooze({ alertId: "backup:gone", conditionKey: "gone", message: "gone soon", until: new Date(now.getTime() + 3 * 3600_000), actor: "Olga" }, now);
|
||||
const reminders = await settleSnoozes([{ id: "disk:disk:server:1:/", message: "web: / is 96% full", severity: "warning", key: "disk:server:1:/" }], later);
|
||||
assert.deepEqual(reminders, [{ key: "disk:server:1:/", message: "web: / is 96% full", severity: "warning" }]);
|
||||
assert.equal((await withoutSnoozed([{ key: "disk:server:1:/", message: "a" }], later)).length, 1);
|
||||
// One whose alert is gone, and has been for half an hour, is dropped quietly; the snooze ending for an alert that went is forgotten without a word.
|
||||
const muchLater = new Date(now.getTime() + 4 * 3600_000);
|
||||
assert.deepEqual(await settleSnoozes([], muchLater), []);
|
||||
assert.equal((await db.select().from(alertSnoozes)).length, 0);
|
||||
assert.equal(await clearSnooze("disk:disk:server:1:/"), null);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user