Bulk snooze, alert rules judged on report, photos on notes
Bulk snooze: Snooze... and Wake on the ticked alerts, with undo that puts back each alert's earlier snooze exactly. Alert rules are judged for a server right after its agent reports, instead of at the next 15-minute pass; only that server's rule findings change, and the check is serialized with the pass so neither overwrites the other's state. Photos on server and device notes: up to 12 each, shrunk in the browser, accepted by their own bytes (JPEG/PNG/GIF/WebP, 4 MB), kept in the database and removed with the server or device. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
251a4a9d73
commit
a16996d975
27 files changed
+8442
-54
No files matched your search
@@ -0,0 +1,173 @@
|
||||
/**
|
||||
* The alert rules are judged right after a server reports, not only at the next pass: announced once, recovered when the figure comes back, only for the server that reported,
|
||||
* leaving everything else in the health state alone, and not run over the top of a pass.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-rules-report-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { servers, serverMetrics, alertRules } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { checkRulesFor, runHealthCheck, RULE_CHECK_MIN_GAP_MS } = await import("../src/services/healthMonitor.js");
|
||||
const { findingKey, computeRuleFindings, listRules } = await import("../src/services/alertRules.js");
|
||||
const { HEALTH_STATE_FLAG } = await import("../src/services/dependencies.js");
|
||||
const { getInternalFlag, setInternalFlag, updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { agentReportRouter } = await import("../src/routes/agentReport.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const messages: string[] = [];
|
||||
let hook: Server;
|
||||
const ids: Record<string, number> = {};
|
||||
let ruleId = 0;
|
||||
|
||||
const state = async () => JSON.parse((await getInternalFlag(HEALTH_STATE_FLAG)) ?? "{}") as Record<string, { message: string; source: string }>;
|
||||
const cpu = async (server: string, value: number, agoMin = 1) => db.insert(serverMetrics).values({ serverId: ids[server], metric: "cpu", value, recordedAt: new Date(Date.now() - agoMin * MIN).toISOString() });
|
||||
const keyFor = (server: string) => findingKey({ ruleId, serverId: ids[server], detail: null });
|
||||
const settle = () => new Promise((r) => setTimeout(r, 150));
|
||||
|
||||
before(async () => {
|
||||
hook = createHttpServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
|
||||
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
|
||||
for (const name of ["web", "pi"]) {
|
||||
ids[name] = (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
}
|
||||
ruleId = (await db.insert(alertRules).values({ name: "CPU hot", enabled: true, metric: "cpu", mount: null, operator: "above", threshold: 90, forMinutes: 0, severity: "warning", serverId: null, tag: null, createdBy: null, createdAt: new Date().toISOString() }).returning({ id: alertRules.id }))[0].id;
|
||||
});
|
||||
after(() => {
|
||||
hook.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
describe("rules judged when a server reports", () => {
|
||||
test("a server that reports a figure past the limit is announced at once, and only that server; what else is in the state is left alone", async () => {
|
||||
await setInternalFlag(HEALTH_STATE_FLAG, JSON.stringify({ "disk:server:999": { message: "elsewhere: disk is 95% full", source: "server" } }));
|
||||
await cpu("web", 97);
|
||||
await cpu("pi", 97); // pi is just as hot, but it has not reported just now
|
||||
const r = await checkRulesFor(ids.web, Date.now(), 0);
|
||||
assert.deepEqual(r, { added: 1, resolved: 0 });
|
||||
const s = await state();
|
||||
assert.ok(keyFor("web") in s);
|
||||
assert.equal(keyFor("pi") in s, false, "the other server waits for its own report or the pass");
|
||||
assert.equal(s["disk:server:999"].message, "elsewhere: disk is 95% full", "what else was known is kept exactly");
|
||||
await settle();
|
||||
assert.equal(messages.filter((m) => /web: CPU/i.test(m) && /CPU hot/.test(m)).length, 1, "one message, naming the rule");
|
||||
});
|
||||
|
||||
test("the same figure again is not announced again", async () => {
|
||||
await cpu("web", 98, 0);
|
||||
const before = messages.length;
|
||||
assert.deepEqual(await checkRulesFor(ids.web, Date.now(), 0), { added: 0, resolved: 0 });
|
||||
await settle();
|
||||
assert.equal(messages.length, before);
|
||||
});
|
||||
|
||||
test("when the next report brings the figure back under the limit it is recovered", async () => {
|
||||
await db.delete(serverMetrics).where(eq(serverMetrics.serverId, ids.web));
|
||||
await cpu("web", 40, 0);
|
||||
const before = messages.length;
|
||||
assert.deepEqual(await checkRulesFor(ids.web, Date.now(), 0), { added: 0, resolved: 1 });
|
||||
assert.equal(keyFor("web") in (await state()), false);
|
||||
assert.equal((await state())["disk:server:999"].source, "server", "still untouched");
|
||||
await settle();
|
||||
assert.ok(messages.length > before, "the recovery is said");
|
||||
});
|
||||
|
||||
test("a server looked at in the last minute is not looked at again; a restart's grace and a server no rule covers cost nothing", async () => {
|
||||
assert.equal(RULE_CHECK_MIN_GAP_MS, 60_000);
|
||||
await cpu("pi", 99, 0);
|
||||
const first = await checkRulesFor(ids.pi, Date.now(), 60_000);
|
||||
assert.equal(first?.added, 1);
|
||||
assert.equal(await checkRulesFor(ids.pi, Date.now() + 10_000, 60_000), null, "ten seconds later");
|
||||
assert.notEqual(await checkRulesFor(ids.pi, Date.now() + 90_000, 60_000), null, "a minute and a half later");
|
||||
await db.update(alertRules).set({ enabled: false }).where(eq(alertRules.id, ruleId));
|
||||
const off = await checkRulesFor(ids.pi, Date.now() + 200_000, 0);
|
||||
assert.equal(off?.resolved, 1, "a rule that was switched off no longer finds it: recovered");
|
||||
await db.update(alertRules).set({ enabled: true }).where(eq(alertRules.id, ruleId));
|
||||
});
|
||||
|
||||
test("only the one server's history is read: the findings are the same as for all, narrowed to it", async () => {
|
||||
await db.delete(serverMetrics);
|
||||
await cpu("web", 95);
|
||||
await cpu("pi", 95);
|
||||
const rules = await listRules();
|
||||
const all = await computeRuleFindings(rules, Date.now());
|
||||
const one = await computeRuleFindings(rules, Date.now(), ids.web);
|
||||
assert.deepEqual(all.map((f) => f.serverId).sort(), [ids.web, ids.pi].sort());
|
||||
assert.deepEqual(one.map((f) => f.serverId), [ids.web]);
|
||||
});
|
||||
|
||||
test("a pass and a check after a report at the same moment do not lose each other's changes", async () => {
|
||||
await db.delete(serverMetrics);
|
||||
await setInternalFlag(HEALTH_STATE_FLAG, "{}");
|
||||
await cpu("web", 97);
|
||||
await cpu("pi", 97);
|
||||
await db.update(servers).set({ lastSeenAt: new Date().toISOString() });
|
||||
await Promise.all([runHealthCheck(Date.now()), checkRulesFor(ids.web, Date.now(), 0)]);
|
||||
const s = await state();
|
||||
assert.ok(keyFor("web") in s && keyFor("pi") in s, "both found, whichever went first");
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the agent's report", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
before(async () => {
|
||||
await db.insert(servers).values({ name: "api", apiTokenHash: hashToken("tok-api"), apiTokenPrefix: "tok-api", lastSeenAt: new Date().toISOString() });
|
||||
const a = express();
|
||||
a.use("/api/agent/report", agentReportRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/agent/report`;
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
test("a report that brings the CPU past a rule's limit raises it before the pass would, and the agent gets its answer first", async () => {
|
||||
await setInternalFlag(HEALTH_STATE_FLAG, "{}");
|
||||
const [{ id }] = await db.select({ id: servers.id }).from(servers).where(eq(servers.name, "api"));
|
||||
const res = await fetch(base, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok-api" }, body: JSON.stringify({ system: { cpu: { load_percent: 99 } }, tasks: [] }) });
|
||||
assert.equal(res.status, 202);
|
||||
await new Promise((r) => setTimeout(r, 600));
|
||||
const s = await state();
|
||||
assert.ok(findingKey({ ruleId, serverId: id, detail: null }) in s, "judged on the report");
|
||||
assert.match(s[findingKey({ ruleId, serverId: id, detail: null })].message, /api: .*99%.*[CPU hot]/);
|
||||
});
|
||||
|
||||
test("a report that is refused does not set anything off", async () => {
|
||||
await setInternalFlag(HEALTH_STATE_FLAG, "{}");
|
||||
const res = await fetch(base, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer wrong" }, body: JSON.stringify({ system: { cpu: { load_percent: 99 } }, tasks: [] }) });
|
||||
assert.equal(res.status, 401);
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
assert.deepEqual(await state(), {});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user