What is normal for the time: baselines per hour of the week
Each server's CPU load and memory use get a baseline for every hour of the week (median and robust spread over up to ten weeks, four needed). A clearly high last hour that never dropped back toward normal is a warning; everything else stays quiet. Server History -> Trends draws a usual line and band. Setting under Notifications, on by default. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
232a5661ab
commit
31044b2b64
17 files changed
+619
-10
No files matched your search
@@ -0,0 +1,248 @@
|
||||
/**
|
||||
* What is normal for a server at this time: the hours of the week and their baselines, when a figure is clearly past them (and when it is not), what is found from the stored
|
||||
* history, how it becomes an alert, and the "usual" the server's page draws.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-baselines-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, serverMetrics, serverMetricsRollup } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const b = await import("../src/services/baselines.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { getSettings, updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { serversRouter } = await import("../src/routes/servers.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const HOUR = 3_600_000;
|
||||
const DAY = 86_400_000;
|
||||
const WEEK = 7 * DAY;
|
||||
/** A Tuesday, 03:40 UTC. */
|
||||
const NOW = Date.parse("2026-10-20T03:40:00Z");
|
||||
const TUE_3 = Date.parse("2026-10-20T03:00:00Z");
|
||||
|
||||
describe("hours of the week", () => {
|
||||
test("a moment is a weekday and an hour in the time zone, summer time included", () => {
|
||||
assert.deepEqual(b.slotOf(NOW, "UTC"), { dow: 2, hour: 3 });
|
||||
assert.deepEqual(b.slotOf(NOW, "Europe/Stockholm"), { dow: 2, hour: 5 }, "UTC+2 on this date");
|
||||
assert.deepEqual(b.slotOf(NOW, "America/New_York"), { dow: 1, hour: 23 }, "still Monday evening there");
|
||||
assert.deepEqual(b.slotOf(Date.parse("2026-01-15T23:30:00Z"), "Europe/Stockholm"), { dow: 5, hour: 0 }, "UTC+1 in winter: already Friday");
|
||||
assert.deepEqual(b.slotOf(NOW, "Not/AZone"), { dow: 2, hour: 3 }, "an unknown zone is read as UTC");
|
||||
assert.equal(b.slotWords({ dow: 2, hour: 3 }), "a Tuesday at 03:00");
|
||||
});
|
||||
|
||||
test("the median, and a spread that one odd week does not stretch", () => {
|
||||
assert.equal(b.median([3, 1, 2]), 2);
|
||||
assert.equal(b.median([4, 1, 2, 3]), 2.5);
|
||||
assert.ok(Number.isNaN(b.median([])));
|
||||
const calm = b.robustSpread([8, 9, 7, 8, 10, 8]);
|
||||
const withOdd = b.robustSpread([8, 9, 7, 8, 10, 95]);
|
||||
assert.ok(calm > 0 && Math.abs(calm - withOdd) < 1.5, `an outlier barely moves it (${calm} vs ${withOdd})`);
|
||||
assert.equal(b.robustSpread([5, 5, 5, 5]), 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the baseline of each hour", () => {
|
||||
const row = (weeksBack: number, v: number, n = 4, hourOfWeek = TUE_3) => ({ t: hourOfWeek - weeksBack * WEEK, v, n });
|
||||
|
||||
test("it needs four weeks of that hour, and says nothing before", () => {
|
||||
const three = b.slotTable([row(1, 8), row(2, 9), row(3, 7)], "UTC", NOW);
|
||||
assert.equal(three[2 * 24 + 3], null);
|
||||
const four = b.slotTable([row(1, 8), row(2, 9), row(3, 7), row(4, 8)], "UTC", NOW);
|
||||
assert.deepEqual([four[2 * 24 + 3]?.median, four[2 * 24 + 3]?.weeks, four[2 * 24 + 3]?.dow, four[2 * 24 + 3]?.hour], [8, 4, 2, 3]);
|
||||
assert.equal(four.filter((s) => s !== null).length, 1, "only that hour has a baseline");
|
||||
assert.equal(four.length, 168);
|
||||
});
|
||||
|
||||
test("the hour in progress, thin hours and anything older than ten weeks are left out", () => {
|
||||
const rows = [row(1, 8), row(2, 9), row(3, 7), row(4, 8), { t: TUE_3, v: 99, n: 4 }, row(5, 50, 1), row(11, 60)];
|
||||
const slot = b.slotTable(rows, "UTC", NOW)[2 * 24 + 3]!;
|
||||
assert.deepEqual([slot.weeks, slot.median], [4, 8], "the 99 of this hour, the one-sample hour and the eleven-week-old one don't count");
|
||||
});
|
||||
|
||||
test("the hours are those of the time zone", () => {
|
||||
const rows = [1, 2, 3, 4].map((w) => row(w, 20));
|
||||
const table = b.slotTable(rows, "Europe/Stockholm", NOW);
|
||||
assert.ok(table[2 * 24 + 5], "03:00 UTC is Tuesday 05:00 in Stockholm");
|
||||
assert.equal(table[2 * 24 + 3], null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("when it is clearly unusual", () => {
|
||||
const slot = { dow: 2, hour: 3, median: 8, spread: 2, weeks: 6 };
|
||||
const recent = (values: number[], newestAgoMin = 3) => values.map((v, i) => ({ t: NOW - (newestAgoMin + (values.length - 1 - i) * 10) * 60_000, v }));
|
||||
const judge = (metric: "cpu" | "mem", values: number[], s: typeof slot | null = slot, ago = 3) => b.judgeUnusual(metric, recent(values, ago), s, NOW);
|
||||
|
||||
test("a busy hour, far above the usual and never back toward it, is said", () => {
|
||||
const u = judge("cpu", [78, 82, 80, 76])!;
|
||||
assert.ok(u);
|
||||
assert.deepEqual([Math.round(u.value), u.usual, u.weeks, u.slot], [79, 8, 6, { dow: 2, hour: 3 }]);
|
||||
assert.equal(b.describeUnusual("wiki-host", u), "wiki-host: CPU load is unusually high for a Tuesday at 03:00 — it averaged 79% over the last hour, where it is normally about 8% (from 6 weeks of this hour)");
|
||||
});
|
||||
|
||||
test("memory has its own, smaller, bar", () => {
|
||||
const mem = { ...slot, median: 40, spread: 1 };
|
||||
assert.ok(judge("mem", [82, 84, 83], mem), "40 → 83");
|
||||
assert.equal(judge("mem", [66, 68, 67], mem), null, "under the 70 % floor, however far above the usual");
|
||||
assert.equal(judge("mem", [72, 74, 73], { ...mem, median: 65 }), null, "only 8 points above");
|
||||
});
|
||||
|
||||
test("it stays quiet unless everything agrees", () => {
|
||||
assert.equal(judge("cpu", [78, 82, 80, 76], null), null, "no baseline for the hour yet");
|
||||
assert.equal(judge("cpu", [78, 82]), null, "two samples is not an hour");
|
||||
assert.equal(judge("cpu", [78, 82, 80], slot, 40), null, "the newest sample is too old");
|
||||
assert.equal(judge("cpu", [46, 48, 49, 47]), null, "below the 50 % floor");
|
||||
assert.equal(judge("cpu", [30, 33, 31, 32]), null, "high for this hour but not far enough above the usual");
|
||||
assert.equal(judge("cpu", [78, 82, 80, 76], { ...slot, spread: 20 }), null, "an hour that is always unpredictable has a wide spread; 71 above is under four of them");
|
||||
assert.equal(judge("cpu", [90, 92, 20, 91]), null, "it dropped back to normal in that hour: a spike");
|
||||
assert.equal(judge("cpu", [5, 6, 4, 7]), null, "low is never said");
|
||||
assert.equal(judge("cpu", [78, 82, 80, 76].map((v) => v), { ...slot, median: 70, spread: 3 }), null, "a server that is always busy at this hour");
|
||||
});
|
||||
|
||||
test("only the last hour counts", () => {
|
||||
const old = [{ t: NOW - 3 * HOUR, v: 95 }, { t: NOW - 2 * HOUR, v: 95 }, { t: NOW - 90 * 60_000, v: 95 }];
|
||||
assert.equal(b.judgeUnusual("cpu", old, slot, NOW), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("found from the stored history", () => {
|
||||
let busy = 0;
|
||||
let young = 0;
|
||||
let away = 0;
|
||||
|
||||
/** Hourly rows for a Tuesday 03:00 and some other hours, `weeks` weeks back. */
|
||||
const seedHours = async (serverId: number, weeks: number, cpu: number, mem: number, at = TUE_3) => {
|
||||
for (let w = 1; w <= weeks; w++) {
|
||||
for (const [metric, v] of [["cpu", cpu + (w % 3) - 1], ["mem", mem + (w % 2)]] as const) {
|
||||
await db.insert(serverMetricsRollup).values({ serverId, metric, grain: "h", at: new Date(at - w * WEEK).toISOString(), sum: v * 4, n: 4, min: v - 1, max: v + 1 });
|
||||
}
|
||||
}
|
||||
};
|
||||
const seedRecent = async (serverId: number, cpu: number[], mem: number[], nowMs = NOW) => {
|
||||
for (let i = 0; i < cpu.length; i++) {
|
||||
const at = new Date(nowMs - (3 + (cpu.length - 1 - i) * 10) * 60_000).toISOString();
|
||||
await db.insert(serverMetrics).values([{ serverId, metric: "cpu", value: cpu[i], recordedAt: at }, { serverId, metric: "mem", value: mem[i], recordedAt: at }]);
|
||||
}
|
||||
};
|
||||
|
||||
before(async () => {
|
||||
const mk = async (name: string, seen: number) => (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, lastSeenAt: new Date(seen).toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
busy = await mk("wiki-host", NOW - 3 * 60_000);
|
||||
young = await mk("new-box", NOW - 3 * 60_000);
|
||||
away = await mk("quiet-box", NOW - 3 * HOUR);
|
||||
await seedHours(busy, 6, 8, 40);
|
||||
await seedRecent(busy, [78, 82, 80, 76], [42, 41, 43, 42]);
|
||||
await seedHours(young, 3, 8, 40);
|
||||
await seedRecent(young, [78, 82, 80, 76], [42, 41, 43, 42]);
|
||||
await seedHours(away, 6, 8, 40);
|
||||
await seedRecent(away, [78, 82, 80, 76], [42, 41, 43, 42]);
|
||||
});
|
||||
|
||||
test("a server that is busy for the hour is found; one with too little history, or not reporting, is not", async () => {
|
||||
const found = await b.computeBaselineFindings("UTC", NOW);
|
||||
assert.deepEqual(found.map((f) => [f.serverName, f.metric, b.baselineKey(f)]), [["wiki-host", "cpu", `baseline:cpu:server:${busy}`]], "memory is normal for the hour, so only the CPU");
|
||||
assert.match(found[0].message, /^wiki-host: CPU load is unusually high for a Tuesday at 03:00 — it averaged 79% over the last hour, where it is normally about \d+% \(from 6 weeks of this hour\)$/);
|
||||
});
|
||||
|
||||
test("the same load at a time it is usual says nothing", async () => {
|
||||
// Wednesday 03:40 is judged against Wednesdays, which have no history here
|
||||
assert.deepEqual(await b.computeBaselineFindings("UTC", NOW + DAY), []);
|
||||
});
|
||||
|
||||
test("the page's view has every hour of the week for each figure, and where it is now", async () => {
|
||||
const v = await b.baselineView(busy, "UTC", NOW);
|
||||
assert.deepEqual([v.timezone, v.now, v.slots.cpu.length, v.slots.mem.length], ["UTC", { dow: 2, hour: 3 }, 168, 168]);
|
||||
assert.deepEqual([v.slots.cpu[2 * 24 + 3]?.weeks, Math.round(v.slots.cpu[2 * 24 + 3]!.median)], [6, 8]);
|
||||
assert.equal(v.slots.cpu[0], null);
|
||||
assert.equal((await b.baselineView(young, "UTC", NOW)).slots.cpu[2 * 24 + 3], null, "three weeks is not enough");
|
||||
});
|
||||
|
||||
test("a finding is an ordinary health condition, left to the server's own offline alert when it is offline, and kept off by the setting", async () => {
|
||||
const found = await b.computeBaselineFindings("UTC", NOW);
|
||||
const snap = { servers: [{ id: busy, name: "wiki-host", lastSeenAt: new Date(NOW - 3 * 60_000).toISOString(), disks: [] }], proxmox: [], synology: [], baselines: found };
|
||||
const list = evaluateHealth(snap, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, NOW);
|
||||
const c = list.find((x) => x.key === `baseline:cpu:server:${busy}`)!;
|
||||
assert.deepEqual([c.source, c.severity], ["server:baseline", "warning"]);
|
||||
assert.equal(subjectOfConditionKey(c.key), `server:${busy}`, "maintenance on the server silences it");
|
||||
const gone = evaluateHealth({ ...snap, servers: [{ ...snap.servers[0], lastSeenAt: new Date(NOW - 5 * HOUR).toISOString() }] }, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, NOW);
|
||||
assert.deepEqual(gone.map((x) => x.key), [`offline:server:${busy}`], "a server that stopped reporting is just offline");
|
||||
assert.equal(evaluateHealth(snap, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, NOW, { skipServers: true }).some((x) => x.key.startsWith("baseline:")), false);
|
||||
|
||||
const checks = (await getSettings()).healthChecks;
|
||||
assert.equal(checks.baselineAlerts, true, "on by default");
|
||||
await updateSettings({ healthChecks: { ...checks, baselineAlerts: false } });
|
||||
assert.equal((await collectSnapshot()).snapshot.baselines, undefined, "off: not even looked for");
|
||||
await updateSettings({ healthChecks: { ...checks, baselineAlerts: true } });
|
||||
});
|
||||
});
|
||||
|
||||
describe("on the Alerts page and the server's page", () => {
|
||||
let id = 0;
|
||||
let app: Server;
|
||||
let base = "";
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
// History relative to the real now, so the hour being judged is the hour it really is.
|
||||
const now = Date.now();
|
||||
const hourStart = Math.floor(now / HOUR) * HOUR;
|
||||
id = (await db.insert(servers).values({ name: "builder", apiTokenHash: "builder", apiTokenPrefix: "b", lastSeenAt: new Date(now - 2 * 60_000).toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
for (let w = 1; w <= 6; w++) await db.insert(serverMetricsRollup).values({ serverId: id, metric: "cpu", grain: "h", at: new Date(hourStart - w * WEEK).toISOString(), sum: 24 + w, n: 4, min: 5, max: 9 });
|
||||
for (let i = 0; i < 4; i++) await db.insert(serverMetrics).values({ serverId: id, metric: "cpu", value: 85 + i, recordedAt: new Date(now - (2 + (3 - i) * 10) * 60_000).toISOString() });
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/servers", serversRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/servers`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("the Alerts page says it, as a warning about the server, with a link to its history", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const a = report.alerts.find((x) => x.id === `monitoring:baseline:cpu:server:${id}`);
|
||||
assert.ok(a, "the alert is there");
|
||||
assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["warning", "monitoring", "Server", `/servers/${id}?view=history`, "server", id]);
|
||||
assert.match(a.message, /^builder: CPU load is unusually high for a \w+day at \d\d:00/);
|
||||
});
|
||||
|
||||
test("the server's page gets every hour of the week, for anyone signed in", async () => {
|
||||
const res = await fetch(`${base}/${id}/baseline`, { headers: { connection: "close" } });
|
||||
assert.equal(res.status, 200);
|
||||
const body = (await res.json()) as any;
|
||||
assert.deepEqual([body.timezone, body.slots.cpu.length, body.slots.mem.length], ["UTC", 168, 168]);
|
||||
const here = body.slots.cpu[body.now.dow * 24 + body.now.hour];
|
||||
assert.deepEqual([here.weeks, Math.round(here.median)], [6, 7]);
|
||||
assert.equal((await fetch(`${base}/999/baseline`, { headers: { connection: "close" } })).status, 404);
|
||||
assert.equal((await fetch(`${base}/abc/baseline`, { headers: { connection: "close" } })).status, 400);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user