What is normal for the time: baselines per hour of the week

Each server's CPU load and memory use get a baseline for every hour of the week
(median and robust spread over up to ten weeks, four needed). A clearly high last
hour that never dropped back toward normal is a warning; everything else stays
quiet. Server History -> Trends draws a usual line and band. Setting under
Notifications, on by default.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-21 21:57:57 +02:00
1 parent 232a5661ab
commit 31044b2b64
17 files changed
+619 -10

No files matched your search

+4 -1
View File
@@ -93,6 +93,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (device) return { type: "unifi", id: Number(device[1]), category: "offline" };
let m = /^rule:\d+:server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: "monitoring" };
m = /^baseline:(?:cpu|mem):server:(\d+)$/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: "monitoring" };
m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : key.startsWith("new-port") ? "monitoring" : "disk" };
m = /^disk:proxmox:(\d+):/.exec(key);
@@ -225,12 +227,13 @@ export async function collectAlerts(): Promise<AlertsReport> {
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : undefined,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
if (held.has("server:rules")) cannotCheck("Alert rules", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi|ups):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
+206
View File
@@ -0,0 +1,206 @@
/**
* What is normal for a server at this time: "this server is unusually busy for a Tuesday at 03:00", judged from its own history rather than a fixed limit.
*
* For each server's CPU load and memory use, every hour of the week (Tuesday 03:00 is one, in the notification time zone) has a baseline: the median of that hour's
* averages over the previous weeks, and a robust spread (the median absolute deviation, so one odd week doesn't stretch it). It needs four weeks of that hour before it says
* anything at all, and uses up to ten (the hourly figures are kept four months).
*
* It is quiet by design. It only speaks about a figure that is *high*, and only when all of these hold: the last hour averaged well past a floor (CPU 50 %, memory 70 %),
* far above the usual (CPU 30 points, memory 15) and at least four spreads above it, and it never dropped back toward normal during that hour — so a spike, a backup that
* always runs a little late, or a server that is simply always busy is not an alert. What it finds is an ordinary health condition (`baseline:<cpu|mem>:server:<id>`), announced,
* recovered, held back, silenced by maintenance and routed like the others. The same baseline draws the "usual" band on a server's trend charts.
*/
import { and, eq, gte, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { serverMetrics, serverMetricsRollup, servers } from "../db/schema.js";
export type BaselineMetric = "cpu" | "mem";
export const BASELINE_METRICS: BaselineMetric[] = ["cpu", "mem"];
export const MIN_WEEKS = 4;
export const MAX_WEEKS = 10;
/** An hour with fewer samples than this is a poor average (the agent was away for most of it) and is left out. */
export const MIN_SAMPLES_PER_HOUR = 2;
/** The last hour is what is judged: at least this many samples, the newest no older than the second number. */
export const RECENT_MINUTES = 60;
export const MIN_RECENT_SAMPLES = 3;
export const MAX_RECENT_AGE_MS = 30 * 60_000;
/** How many spreads above the usual it must be. */
export const SPREADS = 4;
export const RULES: Record<BaselineMetric, { label: string; /** How it reads in a sentence. */ phrase: string; /** It must be at least this high at all. */ floor: number; /** …and this many points above the usual. */ minDelta: number; /** The spread is never taken as less than this. */ minSpread: number }> = {
cpu: { label: "CPU load", phrase: "CPU load", floor: 50, minDelta: 30, minSpread: 4 },
mem: { label: "Memory in use", phrase: "memory use", floor: 70, minDelta: 15, minSpread: 2 },
};
const HOUR_MS = 3_600_000;
const DAY_MS = 86_400_000;
const DAYS = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"];
export interface SlotId {
/** 0 = Sunday … 6 = Saturday, in the time zone. */
dow: number;
hour: number;
}
export interface Slot extends SlotId {
median: number;
/** A robust spread, in the figure's own points. */
spread: number;
/** How many weeks of this hour it stands on. */
weeks: number;
}
const formatters = new Map<string, Intl.DateTimeFormat>();
function formatter(tz: string): Intl.DateTimeFormat {
let f = formatters.get(tz);
if (!f) {
try {
f = new Intl.DateTimeFormat("en-US", { timeZone: tz, weekday: "short", hour: "2-digit", hourCycle: "h23" });
} catch {
f = formatter("UTC");
}
formatters.set(tz, f);
}
return f;
}
const SHORT = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
/** The weekday and hour a moment falls in, in a time zone (UTC if the zone isn't one). */
export function slotOf(ms: number, tz: string): SlotId {
const parts = formatter(tz).formatToParts(new Date(ms));
const weekday = parts.find((p) => p.type === "weekday")?.value ?? "Sun";
const hour = Number(parts.find((p) => p.type === "hour")?.value ?? 0) % 24;
return { dow: Math.max(0, SHORT.indexOf(weekday)), hour };
}
export const slotWords = (s: SlotId) => `a ${DAYS[s.dow]} at ${String(s.hour).padStart(2, "0")}:00`;
export function median(xs: number[]): number {
if (xs.length === 0) return NaN;
const s = [...xs].sort((a, b) => a - b);
const m = Math.floor(s.length / 2);
return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
}
/** The median absolute deviation, scaled to be comparable with a standard deviation. */
export function robustSpread(xs: number[]): number {
const m = median(xs);
return 1.4826 * median(xs.map((x) => Math.abs(x - m)));
}
export interface HourRow {
/** Start of the hour, epoch ms. */
t: number;
/** That hour's average. */
v: number;
/** How many samples it averages. */
n: number;
}
/**
* Pure: the baseline of every hour of the week that has enough weeks behind it (168 entries, null where it hasn't). The hour in progress and anything older than ten weeks
* are left out, as are hours too thinly sampled to be an average.
*/
export function slotTable(rows: HourRow[], tz: string, nowMs: number): (Slot | null)[] {
const byslot: number[][] = Array.from({ length: 168 }, () => []);
const current = Math.floor(nowMs / HOUR_MS) * HOUR_MS;
const oldest = nowMs - MAX_WEEKS * 7 * DAY_MS;
for (const r of rows) {
if (r.t >= current || r.t < oldest || r.n < MIN_SAMPLES_PER_HOUR || !Number.isFinite(r.v)) continue;
const s = slotOf(r.t, tz);
byslot[s.dow * 24 + s.hour].push(r.v);
}
return byslot.map((vals, i) => (vals.length >= MIN_WEEKS ? { dow: Math.floor(i / 24), hour: i % 24, median: median(vals), spread: robustSpread(vals), weeks: vals.length } : null));
}
export interface Unusual {
metric: BaselineMetric;
/** The last hour's average. */
value: number;
usual: number;
weeks: number;
slot: SlotId;
}
/**
* Pure: whether the last hour was clearly unusual for this hour of the week. `recent` are the samples (epoch ms, percent) newest last or in any order. Null when it isn't, when
* there is not enough to say, and when there is no baseline for the hour yet.
*/
export function judgeUnusual(metric: BaselineMetric, recent: { t: number; v: number }[], baseline: Slot | null, nowMs: number): Unusual | null {
if (!baseline) return null;
const rule = RULES[metric];
const window = recent.filter((s) => s.t >= nowMs - RECENT_MINUTES * 60_000 && s.t <= nowMs);
if (window.length < MIN_RECENT_SAMPLES) return null;
if (nowMs - Math.max(...window.map((s) => s.t)) > MAX_RECENT_AGE_MS) return null;
const avg = window.reduce((n, s) => n + s.v, 0) / window.length;
const lowest = Math.min(...window.map((s) => s.v));
const spread = Math.max(baseline.spread, rule.minSpread);
const delta = avg - baseline.median;
if (avg < rule.floor || delta < rule.minDelta || delta / spread < SPREADS) return null;
// It never came back toward normal in that hour: a state, not a spike.
if (lowest < baseline.median + rule.minDelta / 2) return null;
return { metric, value: avg, usual: baseline.median, weeks: baseline.weeks, slot: { dow: baseline.dow, hour: baseline.hour } };
}
export interface BaselineFinding extends Unusual {
serverId: number;
serverName: string;
message: string;
}
export const baselineKey = (f: Pick<BaselineFinding, "metric" | "serverId">) => `baseline:${f.metric}:server:${f.serverId}`;
export function describeUnusual(serverName: string, u: Unusual): string {
return `${serverName}: ${RULES[u.metric].phrase} is unusually high for ${slotWords(u.slot)} — it averaged ${Math.round(u.value)}% over the last hour, where it is normally about ${Math.round(u.usual)}% (from ${u.weeks} weeks of this hour)`;
}
// ─── From the stored history ────────────────────────────────────────────────
/** A server's hourly averages of one or both figures for the weeks the baseline uses. */
async function hourlyRows(serverId: number, metrics: BaselineMetric[], nowMs: number): Promise<Record<BaselineMetric, HourRow[]>> {
const out: Record<BaselineMetric, HourRow[]> = { cpu: [], mem: [] };
const rows = await db
.select({ metric: serverMetricsRollup.metric, at: serverMetricsRollup.at, sum: serverMetricsRollup.sum, n: serverMetricsRollup.n })
.from(serverMetricsRollup)
.where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "h"), inArray(serverMetricsRollup.metric, metrics), gte(serverMetricsRollup.at, new Date(nowMs - MAX_WEEKS * 7 * DAY_MS).toISOString())));
for (const r of rows) if (r.n > 0 && (r.metric === "cpu" || r.metric === "mem")) out[r.metric].push({ t: Date.parse(r.at), v: r.sum / r.n, n: r.n });
return out;
}
/** What is unusual about each server right now, from its stored history. Judges only servers that reported lately. */
export async function computeBaselineFindings(timeZone: string, nowMs: number = Date.now()): Promise<BaselineFinding[]> {
const out: BaselineFinding[] = [];
for (const s of await db.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt }).from(servers)) {
if (!s.lastSeenAt || nowMs - Date.parse(s.lastSeenAt) > 45 * 60_000) continue;
const recentRows = await db
.select({ metric: serverMetrics.metric, at: serverMetrics.recordedAt, value: serverMetrics.value })
.from(serverMetrics)
.where(and(eq(serverMetrics.serverId, s.id), inArray(serverMetrics.metric, BASELINE_METRICS), gte(serverMetrics.recordedAt, new Date(nowMs - RECENT_MINUTES * 60_000).toISOString())));
if (recentRows.length === 0) continue;
const hours = await hourlyRows(s.id, BASELINE_METRICS, nowMs);
const slot = slotOf(nowMs, timeZone);
for (const metric of BASELINE_METRICS) {
const recent = recentRows.filter((r) => r.metric === metric).map((r) => ({ t: Date.parse(r.at), v: r.value }));
const table = slotTable(hours[metric], timeZone, nowMs);
const unusual = judgeUnusual(metric, recent, table[slot.dow * 24 + slot.hour], nowMs);
if (unusual) out.push({ ...unusual, serverId: s.id, serverName: s.name, message: describeUnusual(s.name, unusual) });
}
}
return out;
}
export interface BaselineView {
timezone: string;
/** Now, as a weekday and hour in that zone. */
now: SlotId;
/** Every hour of the week (Sunday 00:00 first), for each figure; null where there are not enough weeks yet. */
slots: Record<BaselineMetric, (Slot | null)[]>;
}
/** The baselines a server's page draws its "usual" band from. */
export async function baselineView(serverId: number, timeZone: string, nowMs: number = Date.now()): Promise<BaselineView> {
const hours = await hourlyRows(serverId, BASELINE_METRICS, nowMs);
return { timezone: timeZone, now: slotOf(nowMs, timeZone), slots: { cpu: slotTable(hours.cpu, timeZone, nowMs), mem: slotTable(hours.mem, timeZone, nowMs) } };
}
+18 -1
View File
@@ -16,6 +16,7 @@ import { newListeningPorts, newPortMessage, parseBaseline, portKey, type NewPort
import { parseJson, type AgentPort } from "./agentPorts.js";
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
import { computeRuleFindings, findingKey, listRules, type RuleFinding } from "./alertRules.js";
import { baselineKey, computeBaselineFindings, type BaselineFinding } from "./baselines.js";
import { recordStorageSamples } from "./storageHistory.js";
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
@@ -78,6 +79,8 @@ export interface HealthSnapshot {
ups?: UpsSnapshot[];
/** What the administrator's own alert rules find on the servers (services/alertRules.ts). */
rules?: RuleFinding[];
/** What is clearly higher than usual for this hour of the week, from each server's own history (services/baselines.ts). */
baselines?: BaselineFinding[];
}
export interface Thresholds {
serverOfflineMinutes: number;
@@ -167,13 +170,17 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
}
}
// The administrator's own rules: judged only for servers that are reporting (one that is not has its offline alert).
// The administrator's own rules, and what is unusual for the time: judged only for servers that are reporting (one that is not has its offline alert).
if (!opts.skipServers) {
const offlineIds = new Set(out.map((c) => /^offline:server:(\d+)$/.exec(c.key)?.[1]).filter((id): id is string => id !== undefined).map(Number));
for (const f of snapshot.rules ?? []) {
if (offlineIds.has(f.serverId)) continue;
out.push({ key: findingKey(f), source: "server:rules", message: f.message, severity: f.severity });
}
for (const f of snapshot.baselines ?? []) {
if (offlineIds.has(f.serverId)) continue;
out.push({ key: baselineKey(f), source: "server:baseline", message: f.message, severity: "warning" });
}
}
for (const u of snapshot.unifi ?? []) {
@@ -385,6 +392,16 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
held.add("server:rules");
}
// What is unusually high for this hour of the week. A failure here costs only this check.
if (checks.baselineAlerts) {
try {
snapshot.baselines = await computeBaselineFindings((await getSettings()).notifications.timezone || "UTC");
} catch (err) {
console.error("[health] couldn't judge what is usual:", err instanceof Error ? err.message : err);
held.add("server:baseline");
}
}
const rows = await db
.select({ id: integrations.id, name: integrations.name, type: integrations.type })
.from(integrations)
+2
View File
@@ -27,6 +27,8 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `server:${m[1]}`;
m = /^rule:\d+:server:(\d+)(?::|$)/.exec(key);
if (m) return `server:${m[1]}`;
m = /^baseline:(?:cpu|mem):server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^offline:device:(\d+)$/.exec(key);
+3 -1
View File
@@ -121,6 +121,8 @@ export interface HealthCheckSettings {
deviceOfflineMinutes: number;
/** Days before the end of an operating system's support that a server is first flagged on the Alerts page. 0 turns it off. */
osSupportWarnDays: number;
/** A server whose CPU load or memory use over the last hour is clearly higher than is normal for this hour of the week is a warning (services/baselines.ts). */
baselineAlerts: boolean;
}
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
@@ -298,7 +300,7 @@ const DEFAULTS: AppSettings = {
selfBackup: { enabled: true, time: "03:30", keep: 14 },
profilePictures: { gravatar: false, identityProvider: false },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90 },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
consistency: { excludedRanges: ["172.16.0.0/12"] },