What is normal for the time: baselines per hour of the week

Each server's CPU load and memory use get a baseline for every hour of the week
(median and robust spread over up to ten weeks, four needed). A clearly high last
hour that never dropped back toward normal is a warning; everything else stays
quiet. Server History -> Trends draws a usual line and band. Setting under
Notifications, on by default.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-21 21:57:57 +02:00
1 parent 232a5661ab
commit 31044b2b64
17 files changed
+619 -10

No files matched your search

+10
View File
@@ -13,6 +13,7 @@ import { describeDiagnostics, parseDiagnostics } from "../services/agentDiagnost
import { checkThresholds, parseThresholds } from "../services/serverThresholds.js";
import { latestAgentVersions } from "../services/agentVersions.js";
import { getServerMetrics, MAX_RANGE_DAYS } from "../services/serverMetrics.js";
import { baselineView } from "../services/baselines.js";
import { getServerActivity } from "../services/serverActivity.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { ID_PATTERNS } from "../utils/pathSegment.js";
@@ -277,6 +278,15 @@ serversRouter.get("/:id/metrics", asyncHandler(async (req, res) => {
res.json({ days, series: await getServerMetrics(id, days) });
}));
// What is normal for each hour of the week, for the "usual" band on the trend charts (services/baselines.ts).
serversRouter.get("/:id/baseline", asyncHandler(async (req, res) => {
const id = Number(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [server] = await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1);
if (!server) return res.status(404).json({ error: "not_found" });
res.json(await baselineView(id, (await getSettings()).notifications.timezone || "UTC"));
}));
// What has happened to a server: changes to it, the health check seeing it stop reporting or a disk fill, and its maintenance
// windows. Operators and above — the audit log it draws on is theirs to read.
serversRouter.get("/:id/activity", requireRole("operator"), asyncHandler(async (req, res) => {
+1 -1
View File
@@ -144,7 +144,7 @@ const updateSchema = z.object({
.partial()
.optional(),
healthChecks: z
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365) })
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365), baselineAlerts: z.boolean() })
.partial()
.optional(),
quietHours: z
+4 -1
View File
@@ -93,6 +93,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (device) return { type: "unifi", id: Number(device[1]), category: "offline" };
let m = /^rule:\d+:server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: "monitoring" };
m = /^baseline:(?:cpu|mem):server:(\d+)$/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: "monitoring" };
m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : key.startsWith("new-port") ? "monitoring" : "disk" };
m = /^disk:proxmox:(\d+):/.exec(key);
@@ -225,12 +227,13 @@ export async function collectAlerts(): Promise<AlertsReport> {
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : undefined,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
if (held.has("server:rules")) cannotCheck("Alert rules", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi|ups):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
+206
View File
@@ -0,0 +1,206 @@
/**
* What is normal for a server at this time: "this server is unusually busy for a Tuesday at 03:00", judged from its own history rather than a fixed limit.
*
* For each server's CPU load and memory use, every hour of the week (Tuesday 03:00 is one, in the notification time zone) has a baseline: the median of that hour's
* averages over the previous weeks, and a robust spread (the median absolute deviation, so one odd week doesn't stretch it). It needs four weeks of that hour before it says
* anything at all, and uses up to ten (the hourly figures are kept four months).
*
* It is quiet by design. It only speaks about a figure that is *high*, and only when all of these hold: the last hour averaged well past a floor (CPU 50 %, memory 70 %),
* far above the usual (CPU 30 points, memory 15) and at least four spreads above it, and it never dropped back toward normal during that hour — so a spike, a backup that
* always runs a little late, or a server that is simply always busy is not an alert. What it finds is an ordinary health condition (`baseline:<cpu|mem>:server:<id>`), announced,
* recovered, held back, silenced by maintenance and routed like the others. The same baseline draws the "usual" band on a server's trend charts.
*/
import { and, eq, gte, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { serverMetrics, serverMetricsRollup, servers } from "../db/schema.js";
export type BaselineMetric = "cpu" | "mem";
export const BASELINE_METRICS: BaselineMetric[] = ["cpu", "mem"];
export const MIN_WEEKS = 4;
export const MAX_WEEKS = 10;
/** An hour with fewer samples than this is a poor average (the agent was away for most of it) and is left out. */
export const MIN_SAMPLES_PER_HOUR = 2;
/** The last hour is what is judged: at least this many samples, the newest no older than the second number. */
export const RECENT_MINUTES = 60;
export const MIN_RECENT_SAMPLES = 3;
export const MAX_RECENT_AGE_MS = 30 * 60_000;
/** How many spreads above the usual it must be. */
export const SPREADS = 4;
export const RULES: Record<BaselineMetric, { label: string; /** How it reads in a sentence. */ phrase: string; /** It must be at least this high at all. */ floor: number; /** …and this many points above the usual. */ minDelta: number; /** The spread is never taken as less than this. */ minSpread: number }> = {
cpu: { label: "CPU load", phrase: "CPU load", floor: 50, minDelta: 30, minSpread: 4 },
mem: { label: "Memory in use", phrase: "memory use", floor: 70, minDelta: 15, minSpread: 2 },
};
const HOUR_MS = 3_600_000;
const DAY_MS = 86_400_000;
const DAYS = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"];
export interface SlotId {
/** 0 = Sunday … 6 = Saturday, in the time zone. */
dow: number;
hour: number;
}
export interface Slot extends SlotId {
median: number;
/** A robust spread, in the figure's own points. */
spread: number;
/** How many weeks of this hour it stands on. */
weeks: number;
}
const formatters = new Map<string, Intl.DateTimeFormat>();
function formatter(tz: string): Intl.DateTimeFormat {
let f = formatters.get(tz);
if (!f) {
try {
f = new Intl.DateTimeFormat("en-US", { timeZone: tz, weekday: "short", hour: "2-digit", hourCycle: "h23" });
} catch {
f = formatter("UTC");
}
formatters.set(tz, f);
}
return f;
}
const SHORT = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
/** The weekday and hour a moment falls in, in a time zone (UTC if the zone isn't one). */
export function slotOf(ms: number, tz: string): SlotId {
const parts = formatter(tz).formatToParts(new Date(ms));
const weekday = parts.find((p) => p.type === "weekday")?.value ?? "Sun";
const hour = Number(parts.find((p) => p.type === "hour")?.value ?? 0) % 24;
return { dow: Math.max(0, SHORT.indexOf(weekday)), hour };
}
export const slotWords = (s: SlotId) => `a ${DAYS[s.dow]} at ${String(s.hour).padStart(2, "0")}:00`;
export function median(xs: number[]): number {
if (xs.length === 0) return NaN;
const s = [...xs].sort((a, b) => a - b);
const m = Math.floor(s.length / 2);
return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
}
/** The median absolute deviation, scaled to be comparable with a standard deviation. */
export function robustSpread(xs: number[]): number {
const m = median(xs);
return 1.4826 * median(xs.map((x) => Math.abs(x - m)));
}
export interface HourRow {
/** Start of the hour, epoch ms. */
t: number;
/** That hour's average. */
v: number;
/** How many samples it averages. */
n: number;
}
/**
* Pure: the baseline of every hour of the week that has enough weeks behind it (168 entries, null where it hasn't). The hour in progress and anything older than ten weeks
* are left out, as are hours too thinly sampled to be an average.
*/
export function slotTable(rows: HourRow[], tz: string, nowMs: number): (Slot | null)[] {
const byslot: number[][] = Array.from({ length: 168 }, () => []);
const current = Math.floor(nowMs / HOUR_MS) * HOUR_MS;
const oldest = nowMs - MAX_WEEKS * 7 * DAY_MS;
for (const r of rows) {
if (r.t >= current || r.t < oldest || r.n < MIN_SAMPLES_PER_HOUR || !Number.isFinite(r.v)) continue;
const s = slotOf(r.t, tz);
byslot[s.dow * 24 + s.hour].push(r.v);
}
return byslot.map((vals, i) => (vals.length >= MIN_WEEKS ? { dow: Math.floor(i / 24), hour: i % 24, median: median(vals), spread: robustSpread(vals), weeks: vals.length } : null));
}
export interface Unusual {
metric: BaselineMetric;
/** The last hour's average. */
value: number;
usual: number;
weeks: number;
slot: SlotId;
}
/**
* Pure: whether the last hour was clearly unusual for this hour of the week. `recent` are the samples (epoch ms, percent) newest last or in any order. Null when it isn't, when
* there is not enough to say, and when there is no baseline for the hour yet.
*/
export function judgeUnusual(metric: BaselineMetric, recent: { t: number; v: number }[], baseline: Slot | null, nowMs: number): Unusual | null {
if (!baseline) return null;
const rule = RULES[metric];
const window = recent.filter((s) => s.t >= nowMs - RECENT_MINUTES * 60_000 && s.t <= nowMs);
if (window.length < MIN_RECENT_SAMPLES) return null;
if (nowMs - Math.max(...window.map((s) => s.t)) > MAX_RECENT_AGE_MS) return null;
const avg = window.reduce((n, s) => n + s.v, 0) / window.length;
const lowest = Math.min(...window.map((s) => s.v));
const spread = Math.max(baseline.spread, rule.minSpread);
const delta = avg - baseline.median;
if (avg < rule.floor || delta < rule.minDelta || delta / spread < SPREADS) return null;
// It never came back toward normal in that hour: a state, not a spike.
if (lowest < baseline.median + rule.minDelta / 2) return null;
return { metric, value: avg, usual: baseline.median, weeks: baseline.weeks, slot: { dow: baseline.dow, hour: baseline.hour } };
}
export interface BaselineFinding extends Unusual {
serverId: number;
serverName: string;
message: string;
}
export const baselineKey = (f: Pick<BaselineFinding, "metric" | "serverId">) => `baseline:${f.metric}:server:${f.serverId}`;
export function describeUnusual(serverName: string, u: Unusual): string {
return `${serverName}: ${RULES[u.metric].phrase} is unusually high for ${slotWords(u.slot)} — it averaged ${Math.round(u.value)}% over the last hour, where it is normally about ${Math.round(u.usual)}% (from ${u.weeks} weeks of this hour)`;
}
// ─── From the stored history ────────────────────────────────────────────────
/** A server's hourly averages of one or both figures for the weeks the baseline uses. */
async function hourlyRows(serverId: number, metrics: BaselineMetric[], nowMs: number): Promise<Record<BaselineMetric, HourRow[]>> {
const out: Record<BaselineMetric, HourRow[]> = { cpu: [], mem: [] };
const rows = await db
.select({ metric: serverMetricsRollup.metric, at: serverMetricsRollup.at, sum: serverMetricsRollup.sum, n: serverMetricsRollup.n })
.from(serverMetricsRollup)
.where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "h"), inArray(serverMetricsRollup.metric, metrics), gte(serverMetricsRollup.at, new Date(nowMs - MAX_WEEKS * 7 * DAY_MS).toISOString())));
for (const r of rows) if (r.n > 0 && (r.metric === "cpu" || r.metric === "mem")) out[r.metric].push({ t: Date.parse(r.at), v: r.sum / r.n, n: r.n });
return out;
}
/** What is unusual about each server right now, from its stored history. Judges only servers that reported lately. */
export async function computeBaselineFindings(timeZone: string, nowMs: number = Date.now()): Promise<BaselineFinding[]> {
const out: BaselineFinding[] = [];
for (const s of await db.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt }).from(servers)) {
if (!s.lastSeenAt || nowMs - Date.parse(s.lastSeenAt) > 45 * 60_000) continue;
const recentRows = await db
.select({ metric: serverMetrics.metric, at: serverMetrics.recordedAt, value: serverMetrics.value })
.from(serverMetrics)
.where(and(eq(serverMetrics.serverId, s.id), inArray(serverMetrics.metric, BASELINE_METRICS), gte(serverMetrics.recordedAt, new Date(nowMs - RECENT_MINUTES * 60_000).toISOString())));
if (recentRows.length === 0) continue;
const hours = await hourlyRows(s.id, BASELINE_METRICS, nowMs);
const slot = slotOf(nowMs, timeZone);
for (const metric of BASELINE_METRICS) {
const recent = recentRows.filter((r) => r.metric === metric).map((r) => ({ t: Date.parse(r.at), v: r.value }));
const table = slotTable(hours[metric], timeZone, nowMs);
const unusual = judgeUnusual(metric, recent, table[slot.dow * 24 + slot.hour], nowMs);
if (unusual) out.push({ ...unusual, serverId: s.id, serverName: s.name, message: describeUnusual(s.name, unusual) });
}
}
return out;
}
export interface BaselineView {
timezone: string;
/** Now, as a weekday and hour in that zone. */
now: SlotId;
/** Every hour of the week (Sunday 00:00 first), for each figure; null where there are not enough weeks yet. */
slots: Record<BaselineMetric, (Slot | null)[]>;
}
/** The baselines a server's page draws its "usual" band from. */
export async function baselineView(serverId: number, timeZone: string, nowMs: number = Date.now()): Promise<BaselineView> {
const hours = await hourlyRows(serverId, BASELINE_METRICS, nowMs);
return { timezone: timeZone, now: slotOf(nowMs, timeZone), slots: { cpu: slotTable(hours.cpu, timeZone, nowMs), mem: slotTable(hours.mem, timeZone, nowMs) } };
}
+18 -1
View File
@@ -16,6 +16,7 @@ import { newListeningPorts, newPortMessage, parseBaseline, portKey, type NewPort
import { parseJson, type AgentPort } from "./agentPorts.js";
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
import { computeRuleFindings, findingKey, listRules, type RuleFinding } from "./alertRules.js";
import { baselineKey, computeBaselineFindings, type BaselineFinding } from "./baselines.js";
import { recordStorageSamples } from "./storageHistory.js";
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
@@ -78,6 +79,8 @@ export interface HealthSnapshot {
ups?: UpsSnapshot[];
/** What the administrator's own alert rules find on the servers (services/alertRules.ts). */
rules?: RuleFinding[];
/** What is clearly higher than usual for this hour of the week, from each server's own history (services/baselines.ts). */
baselines?: BaselineFinding[];
}
export interface Thresholds {
serverOfflineMinutes: number;
@@ -167,13 +170,17 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
}
}
// The administrator's own rules: judged only for servers that are reporting (one that is not has its offline alert).
// The administrator's own rules, and what is unusual for the time: judged only for servers that are reporting (one that is not has its offline alert).
if (!opts.skipServers) {
const offlineIds = new Set(out.map((c) => /^offline:server:(\d+)$/.exec(c.key)?.[1]).filter((id): id is string => id !== undefined).map(Number));
for (const f of snapshot.rules ?? []) {
if (offlineIds.has(f.serverId)) continue;
out.push({ key: findingKey(f), source: "server:rules", message: f.message, severity: f.severity });
}
for (const f of snapshot.baselines ?? []) {
if (offlineIds.has(f.serverId)) continue;
out.push({ key: baselineKey(f), source: "server:baseline", message: f.message, severity: "warning" });
}
}
for (const u of snapshot.unifi ?? []) {
@@ -385,6 +392,16 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
held.add("server:rules");
}
// What is unusually high for this hour of the week. A failure here costs only this check.
if (checks.baselineAlerts) {
try {
snapshot.baselines = await computeBaselineFindings((await getSettings()).notifications.timezone || "UTC");
} catch (err) {
console.error("[health] couldn't judge what is usual:", err instanceof Error ? err.message : err);
held.add("server:baseline");
}
}
const rows = await db
.select({ id: integrations.id, name: integrations.name, type: integrations.type })
.from(integrations)
+2
View File
@@ -27,6 +27,8 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `server:${m[1]}`;
m = /^rule:\d+:server:(\d+)(?::|$)/.exec(key);
if (m) return `server:${m[1]}`;
m = /^baseline:(?:cpu|mem):server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^offline:device:(\d+)$/.exec(key);
+3 -1
View File
@@ -121,6 +121,8 @@ export interface HealthCheckSettings {
deviceOfflineMinutes: number;
/** Days before the end of an operating system's support that a server is first flagged on the Alerts page. 0 turns it off. */
osSupportWarnDays: number;
/** A server whose CPU load or memory use over the last hour is clearly higher than is normal for this hour of the week is a warning (services/baselines.ts). */
baselineAlerts: boolean;
}
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
@@ -298,7 +300,7 @@ const DEFAULTS: AppSettings = {
selfBackup: { enabled: true, time: "03:30", keep: 14 },
profilePictures: { gravatar: false, identityProvider: false },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90 },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
consistency: { excludedRanges: ["172.16.0.0/12"] },
+248
View File
@@ -0,0 +1,248 @@
/**
* What is normal for a server at this time: the hours of the week and their baselines, when a figure is clearly past them (and when it is not), what is found from the stored
* history, how it becomes an alert, and the "usual" the server's page draws.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Server } from "node:http";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-baselines-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, serverMetrics, serverMetricsRollup } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const b = await import("../src/services/baselines.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { getSettings, updateSettings } = await import("../src/services/settingsStore.js");
const { serversRouter } = await import("../src/routes/servers.js");
await runMigrations();
const HOUR = 3_600_000;
const DAY = 86_400_000;
const WEEK = 7 * DAY;
/** A Tuesday, 03:40 UTC. */
const NOW = Date.parse("2026-10-20T03:40:00Z");
const TUE_3 = Date.parse("2026-10-20T03:00:00Z");
describe("hours of the week", () => {
test("a moment is a weekday and an hour in the time zone, summer time included", () => {
assert.deepEqual(b.slotOf(NOW, "UTC"), { dow: 2, hour: 3 });
assert.deepEqual(b.slotOf(NOW, "Europe/Stockholm"), { dow: 2, hour: 5 }, "UTC+2 on this date");
assert.deepEqual(b.slotOf(NOW, "America/New_York"), { dow: 1, hour: 23 }, "still Monday evening there");
assert.deepEqual(b.slotOf(Date.parse("2026-01-15T23:30:00Z"), "Europe/Stockholm"), { dow: 5, hour: 0 }, "UTC+1 in winter: already Friday");
assert.deepEqual(b.slotOf(NOW, "Not/AZone"), { dow: 2, hour: 3 }, "an unknown zone is read as UTC");
assert.equal(b.slotWords({ dow: 2, hour: 3 }), "a Tuesday at 03:00");
});
test("the median, and a spread that one odd week does not stretch", () => {
assert.equal(b.median([3, 1, 2]), 2);
assert.equal(b.median([4, 1, 2, 3]), 2.5);
assert.ok(Number.isNaN(b.median([])));
const calm = b.robustSpread([8, 9, 7, 8, 10, 8]);
const withOdd = b.robustSpread([8, 9, 7, 8, 10, 95]);
assert.ok(calm > 0 && Math.abs(calm - withOdd) < 1.5, `an outlier barely moves it (${calm} vs ${withOdd})`);
assert.equal(b.robustSpread([5, 5, 5, 5]), 0);
});
});
describe("the baseline of each hour", () => {
const row = (weeksBack: number, v: number, n = 4, hourOfWeek = TUE_3) => ({ t: hourOfWeek - weeksBack * WEEK, v, n });
test("it needs four weeks of that hour, and says nothing before", () => {
const three = b.slotTable([row(1, 8), row(2, 9), row(3, 7)], "UTC", NOW);
assert.equal(three[2 * 24 + 3], null);
const four = b.slotTable([row(1, 8), row(2, 9), row(3, 7), row(4, 8)], "UTC", NOW);
assert.deepEqual([four[2 * 24 + 3]?.median, four[2 * 24 + 3]?.weeks, four[2 * 24 + 3]?.dow, four[2 * 24 + 3]?.hour], [8, 4, 2, 3]);
assert.equal(four.filter((s) => s !== null).length, 1, "only that hour has a baseline");
assert.equal(four.length, 168);
});
test("the hour in progress, thin hours and anything older than ten weeks are left out", () => {
const rows = [row(1, 8), row(2, 9), row(3, 7), row(4, 8), { t: TUE_3, v: 99, n: 4 }, row(5, 50, 1), row(11, 60)];
const slot = b.slotTable(rows, "UTC", NOW)[2 * 24 + 3]!;
assert.deepEqual([slot.weeks, slot.median], [4, 8], "the 99 of this hour, the one-sample hour and the eleven-week-old one don't count");
});
test("the hours are those of the time zone", () => {
const rows = [1, 2, 3, 4].map((w) => row(w, 20));
const table = b.slotTable(rows, "Europe/Stockholm", NOW);
assert.ok(table[2 * 24 + 5], "03:00 UTC is Tuesday 05:00 in Stockholm");
assert.equal(table[2 * 24 + 3], null);
});
});
describe("when it is clearly unusual", () => {
const slot = { dow: 2, hour: 3, median: 8, spread: 2, weeks: 6 };
const recent = (values: number[], newestAgoMin = 3) => values.map((v, i) => ({ t: NOW - (newestAgoMin + (values.length - 1 - i) * 10) * 60_000, v }));
const judge = (metric: "cpu" | "mem", values: number[], s: typeof slot | null = slot, ago = 3) => b.judgeUnusual(metric, recent(values, ago), s, NOW);
test("a busy hour, far above the usual and never back toward it, is said", () => {
const u = judge("cpu", [78, 82, 80, 76])!;
assert.ok(u);
assert.deepEqual([Math.round(u.value), u.usual, u.weeks, u.slot], [79, 8, 6, { dow: 2, hour: 3 }]);
assert.equal(b.describeUnusual("wiki-host", u), "wiki-host: CPU load is unusually high for a Tuesday at 03:00 — it averaged 79% over the last hour, where it is normally about 8% (from 6 weeks of this hour)");
});
test("memory has its own, smaller, bar", () => {
const mem = { ...slot, median: 40, spread: 1 };
assert.ok(judge("mem", [82, 84, 83], mem), "40 → 83");
assert.equal(judge("mem", [66, 68, 67], mem), null, "under the 70 % floor, however far above the usual");
assert.equal(judge("mem", [72, 74, 73], { ...mem, median: 65 }), null, "only 8 points above");
});
test("it stays quiet unless everything agrees", () => {
assert.equal(judge("cpu", [78, 82, 80, 76], null), null, "no baseline for the hour yet");
assert.equal(judge("cpu", [78, 82]), null, "two samples is not an hour");
assert.equal(judge("cpu", [78, 82, 80], slot, 40), null, "the newest sample is too old");
assert.equal(judge("cpu", [46, 48, 49, 47]), null, "below the 50 % floor");
assert.equal(judge("cpu", [30, 33, 31, 32]), null, "high for this hour but not far enough above the usual");
assert.equal(judge("cpu", [78, 82, 80, 76], { ...slot, spread: 20 }), null, "an hour that is always unpredictable has a wide spread; 71 above is under four of them");
assert.equal(judge("cpu", [90, 92, 20, 91]), null, "it dropped back to normal in that hour: a spike");
assert.equal(judge("cpu", [5, 6, 4, 7]), null, "low is never said");
assert.equal(judge("cpu", [78, 82, 80, 76].map((v) => v), { ...slot, median: 70, spread: 3 }), null, "a server that is always busy at this hour");
});
test("only the last hour counts", () => {
const old = [{ t: NOW - 3 * HOUR, v: 95 }, { t: NOW - 2 * HOUR, v: 95 }, { t: NOW - 90 * 60_000, v: 95 }];
assert.equal(b.judgeUnusual("cpu", old, slot, NOW), null);
});
});
describe("found from the stored history", () => {
let busy = 0;
let young = 0;
let away = 0;
/** Hourly rows for a Tuesday 03:00 and some other hours, `weeks` weeks back. */
const seedHours = async (serverId: number, weeks: number, cpu: number, mem: number, at = TUE_3) => {
for (let w = 1; w <= weeks; w++) {
for (const [metric, v] of [["cpu", cpu + (w % 3) - 1], ["mem", mem + (w % 2)]] as const) {
await db.insert(serverMetricsRollup).values({ serverId, metric, grain: "h", at: new Date(at - w * WEEK).toISOString(), sum: v * 4, n: 4, min: v - 1, max: v + 1 });
}
}
};
const seedRecent = async (serverId: number, cpu: number[], mem: number[], nowMs = NOW) => {
for (let i = 0; i < cpu.length; i++) {
const at = new Date(nowMs - (3 + (cpu.length - 1 - i) * 10) * 60_000).toISOString();
await db.insert(serverMetrics).values([{ serverId, metric: "cpu", value: cpu[i], recordedAt: at }, { serverId, metric: "mem", value: mem[i], recordedAt: at }]);
}
};
before(async () => {
const mk = async (name: string, seen: number) => (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, lastSeenAt: new Date(seen).toISOString() }).returning({ id: servers.id }))[0].id;
busy = await mk("wiki-host", NOW - 3 * 60_000);
young = await mk("new-box", NOW - 3 * 60_000);
away = await mk("quiet-box", NOW - 3 * HOUR);
await seedHours(busy, 6, 8, 40);
await seedRecent(busy, [78, 82, 80, 76], [42, 41, 43, 42]);
await seedHours(young, 3, 8, 40);
await seedRecent(young, [78, 82, 80, 76], [42, 41, 43, 42]);
await seedHours(away, 6, 8, 40);
await seedRecent(away, [78, 82, 80, 76], [42, 41, 43, 42]);
});
test("a server that is busy for the hour is found; one with too little history, or not reporting, is not", async () => {
const found = await b.computeBaselineFindings("UTC", NOW);
assert.deepEqual(found.map((f) => [f.serverName, f.metric, b.baselineKey(f)]), [["wiki-host", "cpu", `baseline:cpu:server:${busy}`]], "memory is normal for the hour, so only the CPU");
assert.match(found[0].message, /^wiki-host: CPU load is unusually high for a Tuesday at 03:00 — it averaged 79% over the last hour, where it is normally about \d+% \(from 6 weeks of this hour\)$/);
});
test("the same load at a time it is usual says nothing", async () => {
// Wednesday 03:40 is judged against Wednesdays, which have no history here
assert.deepEqual(await b.computeBaselineFindings("UTC", NOW + DAY), []);
});
test("the page's view has every hour of the week for each figure, and where it is now", async () => {
const v = await b.baselineView(busy, "UTC", NOW);
assert.deepEqual([v.timezone, v.now, v.slots.cpu.length, v.slots.mem.length], ["UTC", { dow: 2, hour: 3 }, 168, 168]);
assert.deepEqual([v.slots.cpu[2 * 24 + 3]?.weeks, Math.round(v.slots.cpu[2 * 24 + 3]!.median)], [6, 8]);
assert.equal(v.slots.cpu[0], null);
assert.equal((await b.baselineView(young, "UTC", NOW)).slots.cpu[2 * 24 + 3], null, "three weeks is not enough");
});
test("a finding is an ordinary health condition, left to the server's own offline alert when it is offline, and kept off by the setting", async () => {
const found = await b.computeBaselineFindings("UTC", NOW);
const snap = { servers: [{ id: busy, name: "wiki-host", lastSeenAt: new Date(NOW - 3 * 60_000).toISOString(), disks: [] }], proxmox: [], synology: [], baselines: found };
const list = evaluateHealth(snap, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, NOW);
const c = list.find((x) => x.key === `baseline:cpu:server:${busy}`)!;
assert.deepEqual([c.source, c.severity], ["server:baseline", "warning"]);
assert.equal(subjectOfConditionKey(c.key), `server:${busy}`, "maintenance on the server silences it");
const gone = evaluateHealth({ ...snap, servers: [{ ...snap.servers[0], lastSeenAt: new Date(NOW - 5 * HOUR).toISOString() }] }, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, NOW);
assert.deepEqual(gone.map((x) => x.key), [`offline:server:${busy}`], "a server that stopped reporting is just offline");
assert.equal(evaluateHealth(snap, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, NOW, { skipServers: true }).some((x) => x.key.startsWith("baseline:")), false);
const checks = (await getSettings()).healthChecks;
assert.equal(checks.baselineAlerts, true, "on by default");
await updateSettings({ healthChecks: { ...checks, baselineAlerts: false } });
assert.equal((await collectSnapshot()).snapshot.baselines, undefined, "off: not even looked for");
await updateSettings({ healthChecks: { ...checks, baselineAlerts: true } });
});
});
describe("on the Alerts page and the server's page", () => {
let id = 0;
let app: Server;
let base = "";
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
// History relative to the real now, so the hour being judged is the hour it really is.
const now = Date.now();
const hourStart = Math.floor(now / HOUR) * HOUR;
id = (await db.insert(servers).values({ name: "builder", apiTokenHash: "builder", apiTokenPrefix: "b", lastSeenAt: new Date(now - 2 * 60_000).toISOString() }).returning({ id: servers.id }))[0].id;
for (let w = 1; w <= 6; w++) await db.insert(serverMetricsRollup).values({ serverId: id, metric: "cpu", grain: "h", at: new Date(hourStart - w * WEEK).toISOString(), sum: 24 + w, n: 4, min: 5, max: 9 });
for (let i = 0; i < 4; i++) await db.insert(serverMetrics).values({ serverId: id, metric: "cpu", value: 85 + i, recordedAt: new Date(now - (2 + (3 - i) * 10) * 60_000).toISOString() });
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/servers", serversRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/servers`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the Alerts page says it, as a warning about the server, with a link to its history", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const a = report.alerts.find((x) => x.id === `monitoring:baseline:cpu:server:${id}`);
assert.ok(a, "the alert is there");
assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["warning", "monitoring", "Server", `/servers/${id}?view=history`, "server", id]);
assert.match(a.message, /^builder: CPU load is unusually high for a \w+day at \d\d:00/);
});
test("the server's page gets every hour of the week, for anyone signed in", async () => {
const res = await fetch(`${base}/${id}/baseline`, { headers: { connection: "close" } });
assert.equal(res.status, 200);
const body = (await res.json()) as any;
assert.deepEqual([body.timezone, body.slots.cpu.length, body.slots.mem.length], ["UTC", 168, 168]);
const here = body.slots.cpu[body.now.dow * 24 + body.now.hour];
assert.deepEqual([here.weeks, Math.round(here.median)], [6, 7]);
assert.equal((await fetch(`${base}/999/baseline`, { headers: { connection: "close" } })).status, 404);
assert.equal((await fetch(`${base}/abc/baseline`, { headers: { connection: "close" } })).status, 400);
});
});