/** * The alert rules are judged right after a server reports, not only at the next pass: announced once, recovered when the figure comes back, only for the server that reported, * leaving everything else in the health state alone, and not run over the top of a pass. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { createServer as createHttpServer, type Server } from "node:http"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-rules-report-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { servers, serverMetrics, alertRules } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { checkRulesFor, runHealthCheck, RULE_CHECK_MIN_GAP_MS } = await import("../src/services/healthMonitor.js"); const { findingKey, computeRuleFindings, listRules } = await import("../src/services/alertRules.js"); const { HEALTH_STATE_FLAG } = await import("../src/services/dependencies.js"); const { getInternalFlag, setInternalFlag, updateSettings } = await import("../src/services/settingsStore.js"); const { agentReportRouter } = await import("../src/routes/agentReport.js"); const { hashToken } = await import("../src/services/tokens.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); const MIN = 60_000; const messages: string[] = []; let hook: Server; const ids: Record = {}; let ruleId = 0; const state = async () => JSON.parse((await getInternalFlag(HEALTH_STATE_FLAG)) ?? "{}") as Record; const cpu = async (server: string, value: number, agoMin = 1) => db.insert(serverMetrics).values({ serverId: ids[server], metric: "cpu", value, recordedAt: new Date(Date.now() - agoMin * MIN).toISOString() }); const keyFor = (server: string) => findingKey({ ruleId, serverId: ids[server], detail: null }); const settle = () => new Promise((r) => setTimeout(r, 150)); before(async () => { hook = createHttpServer((req, res) => { const chunks: Buffer[] = []; req.on("data", (c) => chunks.push(c)); req.on("end", () => { messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content)); res.end("{}"); }); }); await new Promise((r) => hook.listen(0, "127.0.0.1", r)); await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } }); for (const name of ["web", "pi"]) { ids[name] = (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id; } ruleId = (await db.insert(alertRules).values({ name: "CPU hot", enabled: true, metric: "cpu", mount: null, operator: "above", threshold: 90, forMinutes: 0, severity: "warning", serverId: null, tag: null, createdBy: null, createdAt: new Date().toISOString() }).returning({ id: alertRules.id }))[0].id; }); after(() => { hook.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); describe("rules judged when a server reports", () => { test("a server that reports a figure past the limit is announced at once, and only that server; what else is in the state is left alone", async () => { await setInternalFlag(HEALTH_STATE_FLAG, JSON.stringify({ "disk:server:999": { message: "elsewhere: disk is 95% full", source: "server" } })); await cpu("web", 97); await cpu("pi", 97); // pi is just as hot, but it has not reported just now const r = await checkRulesFor(ids.web, Date.now(), 0); assert.deepEqual(r, { added: 1, resolved: 0 }); const s = await state(); assert.ok(keyFor("web") in s); assert.equal(keyFor("pi") in s, false, "the other server waits for its own report or the pass"); assert.equal(s["disk:server:999"].message, "elsewhere: disk is 95% full", "what else was known is kept exactly"); await settle(); assert.equal(messages.filter((m) => /web: CPU/i.test(m) && /CPU hot/.test(m)).length, 1, "one message, naming the rule"); }); test("the same figure again is not announced again", async () => { await cpu("web", 98, 0); const before = messages.length; assert.deepEqual(await checkRulesFor(ids.web, Date.now(), 0), { added: 0, resolved: 0 }); await settle(); assert.equal(messages.length, before); }); test("when the next report brings the figure back under the limit it is recovered", async () => { await db.delete(serverMetrics).where(eq(serverMetrics.serverId, ids.web)); await cpu("web", 40, 0); const before = messages.length; assert.deepEqual(await checkRulesFor(ids.web, Date.now(), 0), { added: 0, resolved: 1 }); assert.equal(keyFor("web") in (await state()), false); assert.equal((await state())["disk:server:999"].source, "server", "still untouched"); await settle(); assert.ok(messages.length > before, "the recovery is said"); }); test("a server looked at in the last minute is not looked at again; a restart's grace and a server no rule covers cost nothing", async () => { assert.equal(RULE_CHECK_MIN_GAP_MS, 60_000); await cpu("pi", 99, 0); const first = await checkRulesFor(ids.pi, Date.now(), 60_000); assert.equal(first?.added, 1); assert.equal(await checkRulesFor(ids.pi, Date.now() + 10_000, 60_000), null, "ten seconds later"); assert.notEqual(await checkRulesFor(ids.pi, Date.now() + 90_000, 60_000), null, "a minute and a half later"); await db.update(alertRules).set({ enabled: false }).where(eq(alertRules.id, ruleId)); const off = await checkRulesFor(ids.pi, Date.now() + 200_000, 0); assert.equal(off?.resolved, 1, "a rule that was switched off no longer finds it: recovered"); await db.update(alertRules).set({ enabled: true }).where(eq(alertRules.id, ruleId)); }); test("only the one server's history is read: the findings are the same as for all, narrowed to it", async () => { await db.delete(serverMetrics); await cpu("web", 95); await cpu("pi", 95); const rules = await listRules(); const all = await computeRuleFindings(rules, Date.now()); const one = await computeRuleFindings(rules, Date.now(), ids.web); assert.deepEqual(all.map((f) => f.serverId).sort(), [ids.web, ids.pi].sort()); assert.deepEqual(one.map((f) => f.serverId), [ids.web]); }); test("a pass and a check after a report at the same moment do not lose each other's changes", async () => { await db.delete(serverMetrics); await setInternalFlag(HEALTH_STATE_FLAG, "{}"); await cpu("web", 97); await cpu("pi", 97); await db.update(servers).set({ lastSeenAt: new Date().toISOString() }); await Promise.all([runHealthCheck(Date.now()), checkRulesFor(ids.web, Date.now(), 0)]); const s = await state(); assert.ok(keyFor("web") in s && keyFor("pi") in s, "both found, whichever went first"); }); }); describe("through the agent's report", () => { let app: Server; let base = ""; before(async () => { await db.insert(servers).values({ name: "api", apiTokenHash: hashToken("tok-api"), apiTokenPrefix: "tok-api", lastSeenAt: new Date().toISOString() }); const a = express(); a.use("/api/agent/report", agentReportRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/agent/report`; }); after(() => app.close()); test("a report that brings the CPU past a rule's limit raises it before the pass would, and the agent gets its answer first", async () => { await setInternalFlag(HEALTH_STATE_FLAG, "{}"); const [{ id }] = await db.select({ id: servers.id }).from(servers).where(eq(servers.name, "api")); const res = await fetch(base, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok-api" }, body: JSON.stringify({ system: { cpu: { load_percent: 99 } }, tasks: [] }) }); assert.equal(res.status, 202); await new Promise((r) => setTimeout(r, 600)); const s = await state(); assert.ok(findingKey({ ruleId, serverId: id, detail: null }) in s, "judged on the report"); assert.match(s[findingKey({ ruleId, serverId: id, detail: null })].message, /api: .*99%.*[CPU hot]/); }); test("a report that is refused does not set anything off", async () => { await setInternalFlag(HEALTH_STATE_FLAG, "{}"); const res = await fetch(base, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer wrong" }, body: JSON.stringify({ system: { cpu: { load_percent: 99 } }, tasks: [] }) }); assert.equal(res.status, 401); await new Promise((r) => setTimeout(r, 200)); assert.deepEqual(await state(), {}); }); });