Bulk snooze, alert rules judged on report, photos on notes
Bulk snooze: Snooze... and Wake on the ticked alerts, with undo that puts back each alert's earlier snooze exactly. Alert rules are judged for a server right after its agent reports, instead of at the next 15-minute pass; only that server's rule findings change, and the check is serialized with the pass so neither overwrites the other's state. Photos on server and device notes: up to 12 each, shrunk in the browser, accepted by their own bytes (JPEG/PNG/GIF/WebP, 4 MB), kept in the database and removed with the server or device. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
251a4a9d73
commit
a16996d975
27 files changed
+8442
-54
No files matched your search
@@ -20,7 +20,7 @@ process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, heartbeats, assets, alertAcks, maintenanceWindows, auditLog } = await import("../src/db/schema.js");
|
||||
const { users, servers, heartbeats, assets, alertAcks, alertSnoozes, maintenanceWindows, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { serversBulkRouter, alertsBulkRouter, inventoryBulkRouter } = await import("../src/routes/bulk.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
@@ -269,8 +269,74 @@ describe("alerts", () => {
|
||||
assert.equal((await listActiveWindows()).length, 0);
|
||||
});
|
||||
|
||||
const hoursAhead = (h: number) => new Date(Date.now() + h * 3_600_000).toISOString();
|
||||
|
||||
test("several are snoozed until the same time; one that has gone is skipped; they are hidden from the live list; undo wakes them", async () => {
|
||||
await db.delete(alertSnoozes);
|
||||
const until = hoursAhead(5);
|
||||
const r = await send("/alerts/bulk", { action: "snooze", alertIds: [...alertIds, "automation:heartbeat:99999"], until });
|
||||
assert.equal(r.status, 200);
|
||||
assert.equal(r.body.changed, 3);
|
||||
assert.deepEqual(r.body.skipped.map((x: { reason: string }) => x.reason), ["it isn't there any more"]);
|
||||
const rows = await db.select().from(alertSnoozes);
|
||||
assert.equal(rows.length, 3);
|
||||
assert.ok(rows.every((x) => x.until === new Date(until).toISOString() && x.snoozedBy === "Olga"));
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
assert.ok(alertIds.every((id) => report.alerts.find((a) => a.id === id)?.snoozed), "each shows as snoozed");
|
||||
assert.ok(r.body.undo.items.every((i: { previous: unknown }) => i.previous === null));
|
||||
|
||||
await send("/alerts/bulk", r.body.undo);
|
||||
assert.equal((await db.select().from(alertSnoozes)).length, 0);
|
||||
});
|
||||
|
||||
test("an earlier snooze is replaced, and undo puts it back exactly — when it ends and who made it", async () => {
|
||||
await db.delete(alertSnoozes);
|
||||
await send("/alerts/bulk", { action: "snooze", alertIds: [alertIds[0]], until: hoursAhead(2) });
|
||||
const [first] = await db.select().from(alertSnoozes);
|
||||
const over = await send("/alerts/bulk", { action: "snooze", alertIds: [alertIds[0], alertIds[1]], until: hoursAhead(30) });
|
||||
assert.equal(over.body.changed, 2);
|
||||
assert.notEqual((await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, alertIds[0])))[0].until, first.until);
|
||||
await send("/alerts/bulk", over.body.undo);
|
||||
const rows = await db.select().from(alertSnoozes);
|
||||
assert.equal(rows.length, 1, "the second alert had none before");
|
||||
assert.deepEqual([rows[0].alertId, rows[0].until, rows[0].snoozedAt, rows[0].snoozedBy], [first.alertId, first.until, first.snoozedAt, first.snoozedBy]);
|
||||
});
|
||||
|
||||
test("snoozes are woken, those that were not snoozed are skipped, and undo puts them back", async () => {
|
||||
await db.delete(alertSnoozes);
|
||||
await send("/alerts/bulk", { action: "snooze", alertIds: [alertIds[0], alertIds[1]], until: hoursAhead(4) });
|
||||
const r = await send("/alerts/bulk", { action: "unsnooze", alertIds });
|
||||
assert.equal(r.body.changed, 2);
|
||||
assert.deepEqual(r.body.skipped.map((x: { reason: string }) => x.reason), ["it wasn't snoozed"]);
|
||||
assert.equal((await db.select().from(alertSnoozes)).length, 0);
|
||||
await send("/alerts/bulk", r.body.undo);
|
||||
assert.equal((await db.select().from(alertSnoozes)).length, 2);
|
||||
await db.delete(alertSnoozes);
|
||||
});
|
||||
|
||||
test("a time that has passed, is too far ahead or is not a time is refused, and nothing changes", async () => {
|
||||
await db.delete(alertSnoozes);
|
||||
for (const until of [hoursAhead(-1), hoursAhead(24 * 91), "tomorrow-ish"]) {
|
||||
const r = await send("/alerts/bulk", { action: "snooze", alertIds, until });
|
||||
assert.equal(r.status, 400, until);
|
||||
assert.ok(typeof r.body.message === "string" && r.body.message.length > 0);
|
||||
}
|
||||
assert.equal((await db.select().from(alertSnoozes)).length, 0);
|
||||
});
|
||||
|
||||
test("undoing to a snooze whose time has already passed drops it rather than announcing the alert at once", async () => {
|
||||
await db.delete(alertSnoozes);
|
||||
const previous = { conditionKey: "heartbeat-late:1", message: "late", until: hoursAhead(-1), snoozedBy: "Olga", snoozedAt: hoursAhead(-5) };
|
||||
await send("/alerts/bulk", { action: "snooze", alertIds: [alertIds[0]], until: hoursAhead(3) });
|
||||
const r = await send("/alerts/bulk", { action: "restore_snoozes", items: [{ alertId: alertIds[0], previous }] });
|
||||
assert.equal(r.status, 200);
|
||||
assert.equal((await db.select().from(alertSnoozes)).length, 0);
|
||||
});
|
||||
|
||||
test("an operator is needed; a viewer is not enough", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await send("/alerts/bulk", { action: "snooze", alertIds, until: hoursAhead(3) })).status, 403);
|
||||
assert.equal((await send("/alerts/bulk", { action: "acknowledge", alertIds })).status, 403);
|
||||
who = "operator";
|
||||
});
|
||||
@@ -330,7 +396,7 @@ describe("what is written down", () => {
|
||||
test("each bulk action is one audit entry saying what and how many, and the change log reads them as sentences", async () => {
|
||||
const entries = await db.select().from(auditLog);
|
||||
const actions = new Set(entries.map((e) => `${e.category}/${e.action}`));
|
||||
for (const a of ["server/bulk_tag", "maintenance/bulk_start", "maintenance/bulk_undo", "server/bulk_runs_on", "alert/bulk_ack", "alert/bulk_unack", "alert/bulk_undo", "inventory/bulk_change"]) assert.ok(actions.has(a), a);
|
||||
for (const a of ["server/bulk_tag", "maintenance/bulk_start", "maintenance/bulk_undo", "server/bulk_runs_on", "alert/bulk_ack", "alert/bulk_unack", "alert/bulk_undo", "alert/bulk_snooze", "alert/bulk_unsnooze", "inventory/bulk_change"]) assert.ok(actions.has(a), a);
|
||||
const tag = entries.find((e) => e.action === "bulk_tag" && JSON.parse(e.detail!).change === "add_tag")!;
|
||||
assert.equal(JSON.parse(tag.detail!).tag, "Nightly");
|
||||
assert.equal(JSON.parse(tag.detail!).changed, 3);
|
||||
@@ -340,6 +406,8 @@ describe("what is written down", () => {
|
||||
assert.ok(titles.some((t) => /^Tag added to 3 servers: Nightly$/.test(t)), "tag");
|
||||
assert.ok(titles.some((t) => /^Maintenance started on several at once/.test(t)), "maintenance");
|
||||
assert.ok(titles.some((t) => /alerts? acknowledged at once$/.test(t)), "ack");
|
||||
assert.ok(titles.some((t) => /alerts? snoozed at once$/.test(t)), "snooze");
|
||||
assert.ok(titles.some((t) => /^Snoozing several alerts at once was undone$/.test(t)), "snooze undo");
|
||||
assert.ok(titles.some((t) => /pieces? of hardware changed at once$/.test(t)), "hardware");
|
||||
assert.ok(titles.some((t) => /was undone$/.test(t)), "undo");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* Photos on notes: what is accepted (told by the bytes, never by the name or the type claimed), the limits, who may add and who may only look, how a picture is served, and that
|
||||
* removing the server or the device removes its photos.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-note-photos-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, integrations, networkDevices, notePhotos, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { sniffImage, MAX_PHOTO_BYTES, MAX_PHOTOS_PER_TARGET, MAX_CAPTION } = await import("../src/services/notePhotos.js");
|
||||
const { notePhotosRouter } = await import("../src/routes/notePhotos.js");
|
||||
const { getChangeLog } = await import("../src/services/changeLog.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const png = (extra = 0) => Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), Buffer.alloc(32 + extra, 7)]);
|
||||
const jpeg = () => Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(32, 3)]);
|
||||
const gif = () => Buffer.concat([Buffer.from("GIF89a", "latin1"), Buffer.alloc(32, 1)]);
|
||||
const webp = () => Buffer.concat([Buffer.from("RIFF", "latin1"), Buffer.from([0x20, 0, 0, 0]), Buffer.from("WEBP", "latin1"), Buffer.alloc(24, 2)]);
|
||||
|
||||
describe("what counts as a picture", () => {
|
||||
test("a JPEG, a PNG, a GIF and a WebP are told by how they begin", () => {
|
||||
assert.equal(sniffImage(jpeg()), "image/jpeg");
|
||||
assert.equal(sniffImage(png()), "image/png");
|
||||
assert.equal(sniffImage(gif()), "image/gif");
|
||||
assert.equal(sniffImage(webp()), "image/webp");
|
||||
});
|
||||
|
||||
test("an SVG, a page, a script, an empty or a truncated file are none of them", () => {
|
||||
for (const bad of ['<svg xmlns="http://www.w3.org/2000/svg"><script>alert(1)</script></svg>', "<!doctype html><title>x</title>", "#!/bin/sh\nrm -rf /", ""]) assert.equal(sniffImage(Buffer.from(bad)), null, bad.slice(0, 20));
|
||||
assert.equal(sniffImage(Buffer.from([0x89, 0x50, 0x4e])), null);
|
||||
assert.equal(sniffImage(Buffer.concat([Buffer.from("RIFF", "latin1"), Buffer.alloc(4), Buffer.from("WAVE", "latin1")])), null, "RIFF, but not WebP");
|
||||
});
|
||||
});
|
||||
|
||||
describe("photos through the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
const ids: Record<string, number> = {};
|
||||
|
||||
const send = async (path: string, init: { method?: string; body?: Buffer | string; type?: string } = {}) => {
|
||||
const res = await fetch(`${base}${path}`, { method: init.method ?? "GET", headers: init.type ? { "content-type": init.type } : undefined, body: init.body });
|
||||
const text = res.headers.get("content-type")?.includes("json") ? await res.json() : null;
|
||||
return { status: res.status, res, body: text as any };
|
||||
};
|
||||
const upload = (target: string, type: "server" | "device", bytes: Buffer, caption = "", claimed = "image/png") =>
|
||||
send(`/api/note-photos?targetType=${type}&targetId=${ids[target]}${caption ? `&caption=${encodeURIComponent(caption)}` : ""}`, { method: "POST", body: bytes, type: claimed });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
ids.rack = (await db.insert(servers).values({ name: "rack-pc", apiTokenHash: "r", apiTokenPrefix: "r" }).returning({ id: servers.id }))[0].id;
|
||||
ids.other = (await db.insert(servers).values({ name: "other", apiTokenHash: "o", apiTokenPrefix: "o" }).returning({ id: servers.id }))[0].id;
|
||||
const [unifi] = await db.insert(integrations).values({ type: "unifi", name: "UniFi", baseUrl: "http://10.0.0.1", config: "{}" }).returning({ id: integrations.id });
|
||||
ids.switch = (await db.insert(networkDevices).values({ integrationId: unifi.id, externalId: "aa:bb:cc:dd:ee:ff", name: "core switch", kind: "switch", firstSeenAt: new Date().toISOString(), updatedAt: new Date().toISOString() }).returning({ id: networkDevices.id }))[0].id;
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/note-photos", notePhotosRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("an operator adds a photo with a caption; it is listed, and served with its own type and a policy that lets it run nothing", async () => {
|
||||
const r = await upload("rack", "server", png(), " The back of the rack ");
|
||||
assert.equal(r.status, 201);
|
||||
assert.deepEqual([r.body.photo.caption, r.body.photo.contentType, r.body.photo.createdBy, r.body.photo.size], ["The back of the rack", "image/png", "Olga", png().length]);
|
||||
const list = await send(`/api/note-photos?targetType=server&targetId=${ids.rack}`);
|
||||
assert.deepEqual(list.body.photos.map((p: { id: number }) => p.id), [r.body.photo.id]);
|
||||
const img = await send(`/api/note-photos/${r.body.photo.id}/image`);
|
||||
assert.equal(img.status, 200);
|
||||
assert.equal(img.res.headers.get("content-type"), "image/png");
|
||||
assert.equal(img.res.headers.get("x-content-type-options"), "nosniff");
|
||||
assert.match(img.res.headers.get("content-security-policy") ?? "", /sandbox/);
|
||||
assert.match(img.res.headers.get("cache-control") ?? "", /immutable/);
|
||||
assert.deepEqual(Buffer.from(await img.res.arrayBuffer()), png(), "exactly the bytes that were sent");
|
||||
});
|
||||
|
||||
test("what the picture is comes from its bytes: a PNG sent as something else is a PNG; a script sent as image/png is refused", async () => {
|
||||
const ok = await upload("rack", "server", jpeg(), "", "application/octet-stream");
|
||||
assert.equal(ok.body.photo.contentType, "image/jpeg");
|
||||
const bad = await upload("rack", "server", Buffer.from('<svg xmlns="http://www.w3.org/2000/svg"><script>alert(1)</script></svg>'), "", "image/png");
|
||||
assert.deepEqual([bad.status, bad.body.error], [400, "not_an_image"]);
|
||||
assert.equal((await upload("rack", "server", Buffer.alloc(0))).status, 400);
|
||||
});
|
||||
|
||||
test("a device's photos are its own, and a photo can be of a device as well as of a server", async () => {
|
||||
const r = await upload("switch", "device", gif(), "Port labels");
|
||||
assert.equal(r.status, 201);
|
||||
assert.equal((await send(`/api/note-photos?targetType=device&targetId=${ids.switch}`)).body.photos.length, 1);
|
||||
assert.equal((await send(`/api/note-photos?targetType=server&targetId=${ids.other}`)).body.photos.length, 0);
|
||||
});
|
||||
|
||||
test("a photo over 4 MB is refused with a plain message; one that is not for anything is not found", async () => {
|
||||
const big = await upload("other", "server", png(MAX_PHOTO_BYTES));
|
||||
assert.equal(big.status, 413);
|
||||
assert.equal(big.body.error, "too_big");
|
||||
const nowhere = await send(`/api/note-photos?targetType=server&targetId=999999`, { method: "POST", body: png(), type: "image/png" });
|
||||
assert.equal(nowhere.status, 404);
|
||||
assert.equal((await send(`/api/note-photos?targetType=thing&targetId=1`, { method: "POST", body: png(), type: "image/png" })).status, 400);
|
||||
});
|
||||
|
||||
test("a thing has at most twelve photos", async () => {
|
||||
await db.delete(notePhotos).where(eq(notePhotos.serverId, ids.other));
|
||||
for (let i = 0; i < MAX_PHOTOS_PER_TARGET; i++) assert.equal((await upload("other", "server", webp())).status, 201);
|
||||
const r = await upload("other", "server", webp());
|
||||
assert.deepEqual([r.status, r.body.error], [400, "too_many"]);
|
||||
});
|
||||
|
||||
test("a caption is kept short and can be changed or cleared; a photo can be removed, and the change log says so", async () => {
|
||||
const [photo] = await db.select().from(notePhotos).where(eq(notePhotos.serverId, ids.rack));
|
||||
const long = await send(`/api/note-photos/${photo.id}`, { method: "PUT", body: JSON.stringify({ caption: "x".repeat(MAX_CAPTION + 50) }), type: "application/json" });
|
||||
assert.equal(long.body.photo.caption.length, MAX_CAPTION);
|
||||
const cleared = await send(`/api/note-photos/${photo.id}`, { method: "PUT", body: JSON.stringify({ caption: " " }), type: "application/json" });
|
||||
assert.equal(cleared.body.photo.caption, null);
|
||||
assert.equal((await send(`/api/note-photos/999999`, { method: "PUT", body: JSON.stringify({ caption: "x" }), type: "application/json" })).status, 404);
|
||||
assert.equal((await send(`/api/note-photos/${photo.id}`, { method: "DELETE" })).status, 204);
|
||||
assert.equal((await send(`/api/note-photos/${photo.id}/image`)).status, 404);
|
||||
assert.equal((await send(`/api/note-photos/${photo.id}`, { method: "DELETE" })).status, 404);
|
||||
const actions = (await db.select().from(auditLog)).map((e) => `${e.category}/${e.action}`);
|
||||
assert.ok(actions.includes("note/photo_add") && actions.includes("note/photo_delete"));
|
||||
const titles = (await getChangeLog({ days: 1, limit: 200 })).entries.map((e) => e.title);
|
||||
assert.ok(titles.some((t) => /^A photo was added to a note on rack-pc$/.test(t)), "added");
|
||||
assert.ok(titles.some((t) => /^A photo was removed from a note on rack-pc$/.test(t)), "removed");
|
||||
});
|
||||
|
||||
test("everyone signed in can look; only an operator can add, caption or remove", async () => {
|
||||
who = "viewer";
|
||||
const [photo] = await db.select().from(notePhotos).where(eq(notePhotos.serverId, ids.other));
|
||||
assert.equal((await send(`/api/note-photos?targetType=server&targetId=${ids.other}`)).status, 200);
|
||||
assert.equal((await send(`/api/note-photos/${photo.id}/image`)).status, 200);
|
||||
assert.equal((await upload("other", "server", png())).status, 403);
|
||||
assert.equal((await send(`/api/note-photos/${photo.id}`, { method: "PUT", body: JSON.stringify({ caption: "x" }), type: "application/json" })).status, 403);
|
||||
assert.equal((await send(`/api/note-photos/${photo.id}`, { method: "DELETE" })).status, 403);
|
||||
who = "operator";
|
||||
});
|
||||
|
||||
test("removing the server or the device removes its photos", async () => {
|
||||
assert.ok((await db.select().from(notePhotos).where(eq(notePhotos.serverId, ids.other))).length > 0);
|
||||
await db.delete(servers).where(eq(servers.id, ids.other));
|
||||
assert.equal((await db.select().from(notePhotos).where(eq(notePhotos.serverId, ids.other))).length, 0);
|
||||
assert.equal((await db.select().from(notePhotos).where(eq(notePhotos.deviceId, ids.switch))).length, 1);
|
||||
await db.delete(networkDevices).where(eq(networkDevices.id, ids.switch));
|
||||
assert.equal((await db.select().from(notePhotos).where(eq(notePhotos.deviceId, ids.switch))).length, 0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,173 @@
|
||||
/**
|
||||
* The alert rules are judged right after a server reports, not only at the next pass: announced once, recovered when the figure comes back, only for the server that reported,
|
||||
* leaving everything else in the health state alone, and not run over the top of a pass.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-rules-report-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { servers, serverMetrics, alertRules } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { checkRulesFor, runHealthCheck, RULE_CHECK_MIN_GAP_MS } = await import("../src/services/healthMonitor.js");
|
||||
const { findingKey, computeRuleFindings, listRules } = await import("../src/services/alertRules.js");
|
||||
const { HEALTH_STATE_FLAG } = await import("../src/services/dependencies.js");
|
||||
const { getInternalFlag, setInternalFlag, updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { agentReportRouter } = await import("../src/routes/agentReport.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const messages: string[] = [];
|
||||
let hook: Server;
|
||||
const ids: Record<string, number> = {};
|
||||
let ruleId = 0;
|
||||
|
||||
const state = async () => JSON.parse((await getInternalFlag(HEALTH_STATE_FLAG)) ?? "{}") as Record<string, { message: string; source: string }>;
|
||||
const cpu = async (server: string, value: number, agoMin = 1) => db.insert(serverMetrics).values({ serverId: ids[server], metric: "cpu", value, recordedAt: new Date(Date.now() - agoMin * MIN).toISOString() });
|
||||
const keyFor = (server: string) => findingKey({ ruleId, serverId: ids[server], detail: null });
|
||||
const settle = () => new Promise((r) => setTimeout(r, 150));
|
||||
|
||||
before(async () => {
|
||||
hook = createHttpServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
|
||||
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
|
||||
for (const name of ["web", "pi"]) {
|
||||
ids[name] = (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
}
|
||||
ruleId = (await db.insert(alertRules).values({ name: "CPU hot", enabled: true, metric: "cpu", mount: null, operator: "above", threshold: 90, forMinutes: 0, severity: "warning", serverId: null, tag: null, createdBy: null, createdAt: new Date().toISOString() }).returning({ id: alertRules.id }))[0].id;
|
||||
});
|
||||
after(() => {
|
||||
hook.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
describe("rules judged when a server reports", () => {
|
||||
test("a server that reports a figure past the limit is announced at once, and only that server; what else is in the state is left alone", async () => {
|
||||
await setInternalFlag(HEALTH_STATE_FLAG, JSON.stringify({ "disk:server:999": { message: "elsewhere: disk is 95% full", source: "server" } }));
|
||||
await cpu("web", 97);
|
||||
await cpu("pi", 97); // pi is just as hot, but it has not reported just now
|
||||
const r = await checkRulesFor(ids.web, Date.now(), 0);
|
||||
assert.deepEqual(r, { added: 1, resolved: 0 });
|
||||
const s = await state();
|
||||
assert.ok(keyFor("web") in s);
|
||||
assert.equal(keyFor("pi") in s, false, "the other server waits for its own report or the pass");
|
||||
assert.equal(s["disk:server:999"].message, "elsewhere: disk is 95% full", "what else was known is kept exactly");
|
||||
await settle();
|
||||
assert.equal(messages.filter((m) => /web: CPU/i.test(m) && /CPU hot/.test(m)).length, 1, "one message, naming the rule");
|
||||
});
|
||||
|
||||
test("the same figure again is not announced again", async () => {
|
||||
await cpu("web", 98, 0);
|
||||
const before = messages.length;
|
||||
assert.deepEqual(await checkRulesFor(ids.web, Date.now(), 0), { added: 0, resolved: 0 });
|
||||
await settle();
|
||||
assert.equal(messages.length, before);
|
||||
});
|
||||
|
||||
test("when the next report brings the figure back under the limit it is recovered", async () => {
|
||||
await db.delete(serverMetrics).where(eq(serverMetrics.serverId, ids.web));
|
||||
await cpu("web", 40, 0);
|
||||
const before = messages.length;
|
||||
assert.deepEqual(await checkRulesFor(ids.web, Date.now(), 0), { added: 0, resolved: 1 });
|
||||
assert.equal(keyFor("web") in (await state()), false);
|
||||
assert.equal((await state())["disk:server:999"].source, "server", "still untouched");
|
||||
await settle();
|
||||
assert.ok(messages.length > before, "the recovery is said");
|
||||
});
|
||||
|
||||
test("a server looked at in the last minute is not looked at again; a restart's grace and a server no rule covers cost nothing", async () => {
|
||||
assert.equal(RULE_CHECK_MIN_GAP_MS, 60_000);
|
||||
await cpu("pi", 99, 0);
|
||||
const first = await checkRulesFor(ids.pi, Date.now(), 60_000);
|
||||
assert.equal(first?.added, 1);
|
||||
assert.equal(await checkRulesFor(ids.pi, Date.now() + 10_000, 60_000), null, "ten seconds later");
|
||||
assert.notEqual(await checkRulesFor(ids.pi, Date.now() + 90_000, 60_000), null, "a minute and a half later");
|
||||
await db.update(alertRules).set({ enabled: false }).where(eq(alertRules.id, ruleId));
|
||||
const off = await checkRulesFor(ids.pi, Date.now() + 200_000, 0);
|
||||
assert.equal(off?.resolved, 1, "a rule that was switched off no longer finds it: recovered");
|
||||
await db.update(alertRules).set({ enabled: true }).where(eq(alertRules.id, ruleId));
|
||||
});
|
||||
|
||||
test("only the one server's history is read: the findings are the same as for all, narrowed to it", async () => {
|
||||
await db.delete(serverMetrics);
|
||||
await cpu("web", 95);
|
||||
await cpu("pi", 95);
|
||||
const rules = await listRules();
|
||||
const all = await computeRuleFindings(rules, Date.now());
|
||||
const one = await computeRuleFindings(rules, Date.now(), ids.web);
|
||||
assert.deepEqual(all.map((f) => f.serverId).sort(), [ids.web, ids.pi].sort());
|
||||
assert.deepEqual(one.map((f) => f.serverId), [ids.web]);
|
||||
});
|
||||
|
||||
test("a pass and a check after a report at the same moment do not lose each other's changes", async () => {
|
||||
await db.delete(serverMetrics);
|
||||
await setInternalFlag(HEALTH_STATE_FLAG, "{}");
|
||||
await cpu("web", 97);
|
||||
await cpu("pi", 97);
|
||||
await db.update(servers).set({ lastSeenAt: new Date().toISOString() });
|
||||
await Promise.all([runHealthCheck(Date.now()), checkRulesFor(ids.web, Date.now(), 0)]);
|
||||
const s = await state();
|
||||
assert.ok(keyFor("web") in s && keyFor("pi") in s, "both found, whichever went first");
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the agent's report", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
before(async () => {
|
||||
await db.insert(servers).values({ name: "api", apiTokenHash: hashToken("tok-api"), apiTokenPrefix: "tok-api", lastSeenAt: new Date().toISOString() });
|
||||
const a = express();
|
||||
a.use("/api/agent/report", agentReportRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/agent/report`;
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
test("a report that brings the CPU past a rule's limit raises it before the pass would, and the agent gets its answer first", async () => {
|
||||
await setInternalFlag(HEALTH_STATE_FLAG, "{}");
|
||||
const [{ id }] = await db.select({ id: servers.id }).from(servers).where(eq(servers.name, "api"));
|
||||
const res = await fetch(base, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok-api" }, body: JSON.stringify({ system: { cpu: { load_percent: 99 } }, tasks: [] }) });
|
||||
assert.equal(res.status, 202);
|
||||
await new Promise((r) => setTimeout(r, 600));
|
||||
const s = await state();
|
||||
assert.ok(findingKey({ ruleId, serverId: id, detail: null }) in s, "judged on the report");
|
||||
assert.match(s[findingKey({ ruleId, serverId: id, detail: null })].message, /api: .*99%.*[CPU hot]/);
|
||||
});
|
||||
|
||||
test("a report that is refused does not set anything off", async () => {
|
||||
await setInternalFlag(HEALTH_STATE_FLAG, "{}");
|
||||
const res = await fetch(base, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer wrong" }, body: JSON.stringify({ system: { cpu: { load_percent: 99 } }, tasks: [] }) });
|
||||
assert.equal(res.status, 401);
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
assert.deepEqual(await state(), {});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user