/** * What is normal for a server at this time: the hours of the week and their baselines, when a figure is clearly past them (and when it is not), what is found from the stored * history, how it becomes an alert, and the "usual" the server's page draws. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Server } from "node:http"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-baselines-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers, serverMetrics, serverMetricsRollup } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const b = await import("../src/services/baselines.js"); const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js"); const { subjectOfConditionKey } = await import("../src/services/maintenance.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { getSettings, updateSettings } = await import("../src/services/settingsStore.js"); const { serversRouter } = await import("../src/routes/servers.js"); await runMigrations(); const HOUR = 3_600_000; const DAY = 86_400_000; const WEEK = 7 * DAY; /** A Tuesday, 03:40 UTC. */ const NOW = Date.parse("2026-10-20T03:40:00Z"); const TUE_3 = Date.parse("2026-10-20T03:00:00Z"); describe("hours of the week", () => { test("a moment is a weekday and an hour in the time zone, summer time included", () => { assert.deepEqual(b.slotOf(NOW, "UTC"), { dow: 2, hour: 3 }); assert.deepEqual(b.slotOf(NOW, "Europe/Stockholm"), { dow: 2, hour: 5 }, "UTC+2 on this date"); assert.deepEqual(b.slotOf(NOW, "America/New_York"), { dow: 1, hour: 23 }, "still Monday evening there"); assert.deepEqual(b.slotOf(Date.parse("2026-01-15T23:30:00Z"), "Europe/Stockholm"), { dow: 5, hour: 0 }, "UTC+1 in winter: already Friday"); assert.deepEqual(b.slotOf(NOW, "Not/AZone"), { dow: 2, hour: 3 }, "an unknown zone is read as UTC"); assert.equal(b.slotWords({ dow: 2, hour: 3 }), "a Tuesday at 03:00"); }); test("the median, and a spread that one odd week does not stretch", () => { assert.equal(b.median([3, 1, 2]), 2); assert.equal(b.median([4, 1, 2, 3]), 2.5); assert.ok(Number.isNaN(b.median([]))); const calm = b.robustSpread([8, 9, 7, 8, 10, 8]); const withOdd = b.robustSpread([8, 9, 7, 8, 10, 95]); assert.ok(calm > 0 && Math.abs(calm - withOdd) < 1.5, `an outlier barely moves it (${calm} vs ${withOdd})`); assert.equal(b.robustSpread([5, 5, 5, 5]), 0); }); }); describe("the baseline of each hour", () => { const row = (weeksBack: number, v: number, n = 4, hourOfWeek = TUE_3) => ({ t: hourOfWeek - weeksBack * WEEK, v, n }); test("it needs four weeks of that hour, and says nothing before", () => { const three = b.slotTable([row(1, 8), row(2, 9), row(3, 7)], "UTC", NOW); assert.equal(three[2 * 24 + 3], null); const four = b.slotTable([row(1, 8), row(2, 9), row(3, 7), row(4, 8)], "UTC", NOW); assert.deepEqual([four[2 * 24 + 3]?.median, four[2 * 24 + 3]?.weeks, four[2 * 24 + 3]?.dow, four[2 * 24 + 3]?.hour], [8, 4, 2, 3]); assert.equal(four.filter((s) => s !== null).length, 1, "only that hour has a baseline"); assert.equal(four.length, 168); }); test("the hour in progress, thin hours and anything older than ten weeks are left out", () => { const rows = [row(1, 8), row(2, 9), row(3, 7), row(4, 8), { t: TUE_3, v: 99, n: 4 }, row(5, 50, 1), row(11, 60)]; const slot = b.slotTable(rows, "UTC", NOW)[2 * 24 + 3]!; assert.deepEqual([slot.weeks, slot.median], [4, 8], "the 99 of this hour, the one-sample hour and the eleven-week-old one don't count"); }); test("the hours are those of the time zone", () => { const rows = [1, 2, 3, 4].map((w) => row(w, 20)); const table = b.slotTable(rows, "Europe/Stockholm", NOW); assert.ok(table[2 * 24 + 5], "03:00 UTC is Tuesday 05:00 in Stockholm"); assert.equal(table[2 * 24 + 3], null); }); }); describe("when it is clearly unusual", () => { const slot = { dow: 2, hour: 3, median: 8, spread: 2, weeks: 6 }; const recent = (values: number[], newestAgoMin = 3) => values.map((v, i) => ({ t: NOW - (newestAgoMin + (values.length - 1 - i) * 10) * 60_000, v })); const judge = (metric: "cpu" | "mem", values: number[], s: typeof slot | null = slot, ago = 3) => b.judgeUnusual(metric, recent(values, ago), s, NOW); test("a busy hour, far above the usual and never back toward it, is said", () => { const u = judge("cpu", [78, 82, 80, 76])!; assert.ok(u); assert.deepEqual([Math.round(u.value), u.usual, u.weeks, u.slot], [79, 8, 6, { dow: 2, hour: 3 }]); assert.equal(b.describeUnusual("wiki-host", u), "wiki-host: CPU load is unusually high for a Tuesday at 03:00 — it averaged 79% over the last hour, where it is normally about 8% (from 6 weeks of this hour)"); }); test("memory has its own, smaller, bar", () => { const mem = { ...slot, median: 40, spread: 1 }; assert.ok(judge("mem", [82, 84, 83], mem), "40 → 83"); assert.equal(judge("mem", [66, 68, 67], mem), null, "under the 70 % floor, however far above the usual"); assert.equal(judge("mem", [72, 74, 73], { ...mem, median: 65 }), null, "only 8 points above"); }); test("it stays quiet unless everything agrees", () => { assert.equal(judge("cpu", [78, 82, 80, 76], null), null, "no baseline for the hour yet"); assert.equal(judge("cpu", [78, 82]), null, "two samples is not an hour"); assert.equal(judge("cpu", [78, 82, 80], slot, 40), null, "the newest sample is too old"); assert.equal(judge("cpu", [46, 48, 49, 47]), null, "below the 50 % floor"); assert.equal(judge("cpu", [30, 33, 31, 32]), null, "high for this hour but not far enough above the usual"); assert.equal(judge("cpu", [78, 82, 80, 76], { ...slot, spread: 20 }), null, "an hour that is always unpredictable has a wide spread; 71 above is under four of them"); assert.equal(judge("cpu", [90, 92, 20, 91]), null, "it dropped back to normal in that hour: a spike"); assert.equal(judge("cpu", [5, 6, 4, 7]), null, "low is never said"); assert.equal(judge("cpu", [78, 82, 80, 76].map((v) => v), { ...slot, median: 70, spread: 3 }), null, "a server that is always busy at this hour"); }); test("only the last hour counts", () => { const old = [{ t: NOW - 3 * HOUR, v: 95 }, { t: NOW - 2 * HOUR, v: 95 }, { t: NOW - 90 * 60_000, v: 95 }]; assert.equal(b.judgeUnusual("cpu", old, slot, NOW), null); }); }); describe("found from the stored history", () => { let busy = 0; let young = 0; let away = 0; /** Hourly rows for a Tuesday 03:00 and some other hours, `weeks` weeks back. */ const seedHours = async (serverId: number, weeks: number, cpu: number, mem: number, at = TUE_3) => { for (let w = 1; w <= weeks; w++) { for (const [metric, v] of [["cpu", cpu + (w % 3) - 1], ["mem", mem + (w % 2)]] as const) { await db.insert(serverMetricsRollup).values({ serverId, metric, grain: "h", at: new Date(at - w * WEEK).toISOString(), sum: v * 4, n: 4, min: v - 1, max: v + 1 }); } } }; const seedRecent = async (serverId: number, cpu: number[], mem: number[], nowMs = NOW) => { for (let i = 0; i < cpu.length; i++) { const at = new Date(nowMs - (3 + (cpu.length - 1 - i) * 10) * 60_000).toISOString(); await db.insert(serverMetrics).values([{ serverId, metric: "cpu", value: cpu[i], recordedAt: at }, { serverId, metric: "mem", value: mem[i], recordedAt: at }]); } }; before(async () => { const mk = async (name: string, seen: number) => (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, lastSeenAt: new Date(seen).toISOString() }).returning({ id: servers.id }))[0].id; busy = await mk("wiki-host", NOW - 3 * 60_000); young = await mk("new-box", NOW - 3 * 60_000); away = await mk("quiet-box", NOW - 3 * HOUR); await seedHours(busy, 6, 8, 40); await seedRecent(busy, [78, 82, 80, 76], [42, 41, 43, 42]); await seedHours(young, 3, 8, 40); await seedRecent(young, [78, 82, 80, 76], [42, 41, 43, 42]); await seedHours(away, 6, 8, 40); await seedRecent(away, [78, 82, 80, 76], [42, 41, 43, 42]); }); test("a server that is busy for the hour is found; one with too little history, or not reporting, is not", async () => { const found = await b.computeBaselineFindings("UTC", NOW); assert.deepEqual(found.map((f) => [f.serverName, f.metric, b.baselineKey(f)]), [["wiki-host", "cpu", `baseline:cpu:server:${busy}`]], "memory is normal for the hour, so only the CPU"); assert.match(found[0].message, /^wiki-host: CPU load is unusually high for a Tuesday at 03:00 — it averaged 79% over the last hour, where it is normally about \d+% \(from 6 weeks of this hour\)$/); }); test("the same load at a time it is usual says nothing", async () => { // Wednesday 03:40 is judged against Wednesdays, which have no history here assert.deepEqual(await b.computeBaselineFindings("UTC", NOW + DAY), []); }); test("the page's view has every hour of the week for each figure, and where it is now", async () => { const v = await b.baselineView(busy, "UTC", NOW); assert.deepEqual([v.timezone, v.now, v.slots.cpu.length, v.slots.mem.length], ["UTC", { dow: 2, hour: 3 }, 168, 168]); assert.deepEqual([v.slots.cpu[2 * 24 + 3]?.weeks, Math.round(v.slots.cpu[2 * 24 + 3]!.median)], [6, 8]); assert.equal(v.slots.cpu[0], null); assert.equal((await b.baselineView(young, "UTC", NOW)).slots.cpu[2 * 24 + 3], null, "three weeks is not enough"); }); test("a finding is an ordinary health condition, left to the server's own offline alert when it is offline, and kept off by the setting", async () => { const found = await b.computeBaselineFindings("UTC", NOW); const snap = { servers: [{ id: busy, name: "wiki-host", lastSeenAt: new Date(NOW - 3 * 60_000).toISOString(), disks: [] }], proxmox: [], synology: [], baselines: found }; const list = evaluateHealth(snap, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, NOW); const c = list.find((x) => x.key === `baseline:cpu:server:${busy}`)!; assert.deepEqual([c.source, c.severity], ["server:baseline", "warning"]); assert.equal(subjectOfConditionKey(c.key), `server:${busy}`, "maintenance on the server silences it"); const gone = evaluateHealth({ ...snap, servers: [{ ...snap.servers[0], lastSeenAt: new Date(NOW - 5 * HOUR).toISOString() }] }, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, NOW); assert.deepEqual(gone.map((x) => x.key), [`offline:server:${busy}`], "a server that stopped reporting is just offline"); assert.equal(evaluateHealth(snap, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, NOW, { skipServers: true }).some((x) => x.key.startsWith("baseline:")), false); const checks = (await getSettings()).healthChecks; assert.equal(checks.baselineAlerts, true, "on by default"); await updateSettings({ healthChecks: { ...checks, baselineAlerts: false } }); assert.equal((await collectSnapshot()).snapshot.baselines, undefined, "off: not even looked for"); await updateSettings({ healthChecks: { ...checks, baselineAlerts: true } }); }); }); describe("on the Alerts page and the server's page", () => { let id = 0; let app: Server; let base = ""; before(async () => { await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]); // History relative to the real now, so the hour being judged is the hour it really is. const now = Date.now(); const hourStart = Math.floor(now / HOUR) * HOUR; id = (await db.insert(servers).values({ name: "builder", apiTokenHash: "builder", apiTokenPrefix: "b", lastSeenAt: new Date(now - 2 * 60_000).toISOString() }).returning({ id: servers.id }))[0].id; for (let w = 1; w <= 6; w++) await db.insert(serverMetricsRollup).values({ serverId: id, metric: "cpu", grain: "h", at: new Date(hourStart - w * WEEK).toISOString(), sum: 24 + w, n: 4, min: 5, max: 9 }); for (let i = 0; i < 4; i++) await db.insert(serverMetrics).values({ serverId: id, metric: "cpu", value: 85 + i, recordedAt: new Date(now - (2 + (3 - i) * 10) * 60_000).toISOString() }); const a = express(); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: "viewer" } }; next(); }); a.use("/api/servers", serversRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/servers`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("the Alerts page says it, as a warning about the server, with a link to its history", async () => { invalidateAlertsCache(); const report = await getAlerts(true); const a = report.alerts.find((x) => x.id === `monitoring:baseline:cpu:server:${id}`); assert.ok(a, "the alert is there"); assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["warning", "monitoring", "Server", `/servers/${id}?view=history`, "server", id]); assert.match(a.message, /^builder: CPU load is unusually high for a \w+day at \d\d:00/); }); test("the server's page gets every hour of the week, for anyone signed in", async () => { const res = await fetch(`${base}/${id}/baseline`, { headers: { connection: "close" } }); assert.equal(res.status, 200); const body = (await res.json()) as any; assert.deepEqual([body.timezone, body.slots.cpu.length, body.slots.mem.length], ["UTC", 168, 168]); const here = body.slots.cpu[body.now.dow * 24 + body.now.hour]; assert.deepEqual([here.weeks, Math.round(here.median)], [6, 7]); assert.equal((await fetch(`${base}/999/baseline`, { headers: { connection: "close" } })).status, 404); assert.equal((await fetch(`${base}/abc/baseline`, { headers: { connection: "close" } })).status, 400); }); });