What is normal for the time: baselines per hour of the week
Each server's CPU load and memory use get a baseline for every hour of the week (median and robust spread over up to ten weeks, four needed). A clearly high last hour that never dropped back toward normal is a warning; everything else stays quiet. Server History -> Trends draws a usual line and band. Setting under Notifications, on by default. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
232a5661ab
commit
31044b2b64
17 files changed
+619
-10
No files matched your search
@@ -93,6 +93,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (device) return { type: "unifi", id: Number(device[1]), category: "offline" };
|
||||
let m = /^rule:\d+:server:(\d+)/.exec(key);
|
||||
if (m) return { type: "server", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^baseline:(?:cpu|mem):server:(\d+)$/.exec(key);
|
||||
if (m) return { type: "server", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
|
||||
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : key.startsWith("new-port") ? "monitoring" : "disk" };
|
||||
m = /^disk:proxmox:(\d+):/.exec(key);
|
||||
@@ -225,12 +227,13 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
|
||||
if (held.has("server:rules")) cannotCheck("Alert rules", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
/**
|
||||
* What is normal for a server at this time: "this server is unusually busy for a Tuesday at 03:00", judged from its own history rather than a fixed limit.
|
||||
*
|
||||
* For each server's CPU load and memory use, every hour of the week (Tuesday 03:00 is one, in the notification time zone) has a baseline: the median of that hour's
|
||||
* averages over the previous weeks, and a robust spread (the median absolute deviation, so one odd week doesn't stretch it). It needs four weeks of that hour before it says
|
||||
* anything at all, and uses up to ten (the hourly figures are kept four months).
|
||||
*
|
||||
* It is quiet by design. It only speaks about a figure that is *high*, and only when all of these hold: the last hour averaged well past a floor (CPU 50 %, memory 70 %),
|
||||
* far above the usual (CPU 30 points, memory 15) and at least four spreads above it, and it never dropped back toward normal during that hour — so a spike, a backup that
|
||||
* always runs a little late, or a server that is simply always busy is not an alert. What it finds is an ordinary health condition (`baseline:<cpu|mem>:server:<id>`), announced,
|
||||
* recovered, held back, silenced by maintenance and routed like the others. The same baseline draws the "usual" band on a server's trend charts.
|
||||
*/
|
||||
import { and, eq, gte, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { serverMetrics, serverMetricsRollup, servers } from "../db/schema.js";
|
||||
|
||||
export type BaselineMetric = "cpu" | "mem";
|
||||
export const BASELINE_METRICS: BaselineMetric[] = ["cpu", "mem"];
|
||||
|
||||
export const MIN_WEEKS = 4;
|
||||
export const MAX_WEEKS = 10;
|
||||
/** An hour with fewer samples than this is a poor average (the agent was away for most of it) and is left out. */
|
||||
export const MIN_SAMPLES_PER_HOUR = 2;
|
||||
/** The last hour is what is judged: at least this many samples, the newest no older than the second number. */
|
||||
export const RECENT_MINUTES = 60;
|
||||
export const MIN_RECENT_SAMPLES = 3;
|
||||
export const MAX_RECENT_AGE_MS = 30 * 60_000;
|
||||
/** How many spreads above the usual it must be. */
|
||||
export const SPREADS = 4;
|
||||
|
||||
export const RULES: Record<BaselineMetric, { label: string; /** How it reads in a sentence. */ phrase: string; /** It must be at least this high at all. */ floor: number; /** …and this many points above the usual. */ minDelta: number; /** The spread is never taken as less than this. */ minSpread: number }> = {
|
||||
cpu: { label: "CPU load", phrase: "CPU load", floor: 50, minDelta: 30, minSpread: 4 },
|
||||
mem: { label: "Memory in use", phrase: "memory use", floor: 70, minDelta: 15, minSpread: 2 },
|
||||
};
|
||||
|
||||
const HOUR_MS = 3_600_000;
|
||||
const DAY_MS = 86_400_000;
|
||||
const DAYS = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"];
|
||||
|
||||
export interface SlotId {
|
||||
/** 0 = Sunday … 6 = Saturday, in the time zone. */
|
||||
dow: number;
|
||||
hour: number;
|
||||
}
|
||||
|
||||
export interface Slot extends SlotId {
|
||||
median: number;
|
||||
/** A robust spread, in the figure's own points. */
|
||||
spread: number;
|
||||
/** How many weeks of this hour it stands on. */
|
||||
weeks: number;
|
||||
}
|
||||
|
||||
const formatters = new Map<string, Intl.DateTimeFormat>();
|
||||
function formatter(tz: string): Intl.DateTimeFormat {
|
||||
let f = formatters.get(tz);
|
||||
if (!f) {
|
||||
try {
|
||||
f = new Intl.DateTimeFormat("en-US", { timeZone: tz, weekday: "short", hour: "2-digit", hourCycle: "h23" });
|
||||
} catch {
|
||||
f = formatter("UTC");
|
||||
}
|
||||
formatters.set(tz, f);
|
||||
}
|
||||
return f;
|
||||
}
|
||||
|
||||
const SHORT = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
|
||||
/** The weekday and hour a moment falls in, in a time zone (UTC if the zone isn't one). */
|
||||
export function slotOf(ms: number, tz: string): SlotId {
|
||||
const parts = formatter(tz).formatToParts(new Date(ms));
|
||||
const weekday = parts.find((p) => p.type === "weekday")?.value ?? "Sun";
|
||||
const hour = Number(parts.find((p) => p.type === "hour")?.value ?? 0) % 24;
|
||||
return { dow: Math.max(0, SHORT.indexOf(weekday)), hour };
|
||||
}
|
||||
|
||||
export const slotWords = (s: SlotId) => `a ${DAYS[s.dow]} at ${String(s.hour).padStart(2, "0")}:00`;
|
||||
|
||||
export function median(xs: number[]): number {
|
||||
if (xs.length === 0) return NaN;
|
||||
const s = [...xs].sort((a, b) => a - b);
|
||||
const m = Math.floor(s.length / 2);
|
||||
return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
|
||||
}
|
||||
|
||||
/** The median absolute deviation, scaled to be comparable with a standard deviation. */
|
||||
export function robustSpread(xs: number[]): number {
|
||||
const m = median(xs);
|
||||
return 1.4826 * median(xs.map((x) => Math.abs(x - m)));
|
||||
}
|
||||
|
||||
export interface HourRow {
|
||||
/** Start of the hour, epoch ms. */
|
||||
t: number;
|
||||
/** That hour's average. */
|
||||
v: number;
|
||||
/** How many samples it averages. */
|
||||
n: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: the baseline of every hour of the week that has enough weeks behind it (168 entries, null where it hasn't). The hour in progress and anything older than ten weeks
|
||||
* are left out, as are hours too thinly sampled to be an average.
|
||||
*/
|
||||
export function slotTable(rows: HourRow[], tz: string, nowMs: number): (Slot | null)[] {
|
||||
const byslot: number[][] = Array.from({ length: 168 }, () => []);
|
||||
const current = Math.floor(nowMs / HOUR_MS) * HOUR_MS;
|
||||
const oldest = nowMs - MAX_WEEKS * 7 * DAY_MS;
|
||||
for (const r of rows) {
|
||||
if (r.t >= current || r.t < oldest || r.n < MIN_SAMPLES_PER_HOUR || !Number.isFinite(r.v)) continue;
|
||||
const s = slotOf(r.t, tz);
|
||||
byslot[s.dow * 24 + s.hour].push(r.v);
|
||||
}
|
||||
return byslot.map((vals, i) => (vals.length >= MIN_WEEKS ? { dow: Math.floor(i / 24), hour: i % 24, median: median(vals), spread: robustSpread(vals), weeks: vals.length } : null));
|
||||
}
|
||||
|
||||
export interface Unusual {
|
||||
metric: BaselineMetric;
|
||||
/** The last hour's average. */
|
||||
value: number;
|
||||
usual: number;
|
||||
weeks: number;
|
||||
slot: SlotId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: whether the last hour was clearly unusual for this hour of the week. `recent` are the samples (epoch ms, percent) newest last or in any order. Null when it isn't, when
|
||||
* there is not enough to say, and when there is no baseline for the hour yet.
|
||||
*/
|
||||
export function judgeUnusual(metric: BaselineMetric, recent: { t: number; v: number }[], baseline: Slot | null, nowMs: number): Unusual | null {
|
||||
if (!baseline) return null;
|
||||
const rule = RULES[metric];
|
||||
const window = recent.filter((s) => s.t >= nowMs - RECENT_MINUTES * 60_000 && s.t <= nowMs);
|
||||
if (window.length < MIN_RECENT_SAMPLES) return null;
|
||||
if (nowMs - Math.max(...window.map((s) => s.t)) > MAX_RECENT_AGE_MS) return null;
|
||||
const avg = window.reduce((n, s) => n + s.v, 0) / window.length;
|
||||
const lowest = Math.min(...window.map((s) => s.v));
|
||||
const spread = Math.max(baseline.spread, rule.minSpread);
|
||||
const delta = avg - baseline.median;
|
||||
if (avg < rule.floor || delta < rule.minDelta || delta / spread < SPREADS) return null;
|
||||
// It never came back toward normal in that hour: a state, not a spike.
|
||||
if (lowest < baseline.median + rule.minDelta / 2) return null;
|
||||
return { metric, value: avg, usual: baseline.median, weeks: baseline.weeks, slot: { dow: baseline.dow, hour: baseline.hour } };
|
||||
}
|
||||
|
||||
export interface BaselineFinding extends Unusual {
|
||||
serverId: number;
|
||||
serverName: string;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export const baselineKey = (f: Pick<BaselineFinding, "metric" | "serverId">) => `baseline:${f.metric}:server:${f.serverId}`;
|
||||
|
||||
export function describeUnusual(serverName: string, u: Unusual): string {
|
||||
return `${serverName}: ${RULES[u.metric].phrase} is unusually high for ${slotWords(u.slot)} — it averaged ${Math.round(u.value)}% over the last hour, where it is normally about ${Math.round(u.usual)}% (from ${u.weeks} weeks of this hour)`;
|
||||
}
|
||||
|
||||
// ─── From the stored history ────────────────────────────────────────────────
|
||||
|
||||
/** A server's hourly averages of one or both figures for the weeks the baseline uses. */
|
||||
async function hourlyRows(serverId: number, metrics: BaselineMetric[], nowMs: number): Promise<Record<BaselineMetric, HourRow[]>> {
|
||||
const out: Record<BaselineMetric, HourRow[]> = { cpu: [], mem: [] };
|
||||
const rows = await db
|
||||
.select({ metric: serverMetricsRollup.metric, at: serverMetricsRollup.at, sum: serverMetricsRollup.sum, n: serverMetricsRollup.n })
|
||||
.from(serverMetricsRollup)
|
||||
.where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "h"), inArray(serverMetricsRollup.metric, metrics), gte(serverMetricsRollup.at, new Date(nowMs - MAX_WEEKS * 7 * DAY_MS).toISOString())));
|
||||
for (const r of rows) if (r.n > 0 && (r.metric === "cpu" || r.metric === "mem")) out[r.metric].push({ t: Date.parse(r.at), v: r.sum / r.n, n: r.n });
|
||||
return out;
|
||||
}
|
||||
|
||||
/** What is unusual about each server right now, from its stored history. Judges only servers that reported lately. */
|
||||
export async function computeBaselineFindings(timeZone: string, nowMs: number = Date.now()): Promise<BaselineFinding[]> {
|
||||
const out: BaselineFinding[] = [];
|
||||
for (const s of await db.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt }).from(servers)) {
|
||||
if (!s.lastSeenAt || nowMs - Date.parse(s.lastSeenAt) > 45 * 60_000) continue;
|
||||
const recentRows = await db
|
||||
.select({ metric: serverMetrics.metric, at: serverMetrics.recordedAt, value: serverMetrics.value })
|
||||
.from(serverMetrics)
|
||||
.where(and(eq(serverMetrics.serverId, s.id), inArray(serverMetrics.metric, BASELINE_METRICS), gte(serverMetrics.recordedAt, new Date(nowMs - RECENT_MINUTES * 60_000).toISOString())));
|
||||
if (recentRows.length === 0) continue;
|
||||
const hours = await hourlyRows(s.id, BASELINE_METRICS, nowMs);
|
||||
const slot = slotOf(nowMs, timeZone);
|
||||
for (const metric of BASELINE_METRICS) {
|
||||
const recent = recentRows.filter((r) => r.metric === metric).map((r) => ({ t: Date.parse(r.at), v: r.value }));
|
||||
const table = slotTable(hours[metric], timeZone, nowMs);
|
||||
const unusual = judgeUnusual(metric, recent, table[slot.dow * 24 + slot.hour], nowMs);
|
||||
if (unusual) out.push({ ...unusual, serverId: s.id, serverName: s.name, message: describeUnusual(s.name, unusual) });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface BaselineView {
|
||||
timezone: string;
|
||||
/** Now, as a weekday and hour in that zone. */
|
||||
now: SlotId;
|
||||
/** Every hour of the week (Sunday 00:00 first), for each figure; null where there are not enough weeks yet. */
|
||||
slots: Record<BaselineMetric, (Slot | null)[]>;
|
||||
}
|
||||
|
||||
/** The baselines a server's page draws its "usual" band from. */
|
||||
export async function baselineView(serverId: number, timeZone: string, nowMs: number = Date.now()): Promise<BaselineView> {
|
||||
const hours = await hourlyRows(serverId, BASELINE_METRICS, nowMs);
|
||||
return { timezone: timeZone, now: slotOf(nowMs, timeZone), slots: { cpu: slotTable(hours.cpu, timeZone, nowMs), mem: slotTable(hours.mem, timeZone, nowMs) } };
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import { newListeningPorts, newPortMessage, parseBaseline, portKey, type NewPort
|
||||
import { parseJson, type AgentPort } from "./agentPorts.js";
|
||||
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
|
||||
import { computeRuleFindings, findingKey, listRules, type RuleFinding } from "./alertRules.js";
|
||||
import { baselineKey, computeBaselineFindings, type BaselineFinding } from "./baselines.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
|
||||
|
||||
@@ -78,6 +79,8 @@ export interface HealthSnapshot {
|
||||
ups?: UpsSnapshot[];
|
||||
/** What the administrator's own alert rules find on the servers (services/alertRules.ts). */
|
||||
rules?: RuleFinding[];
|
||||
/** What is clearly higher than usual for this hour of the week, from each server's own history (services/baselines.ts). */
|
||||
baselines?: BaselineFinding[];
|
||||
}
|
||||
export interface Thresholds {
|
||||
serverOfflineMinutes: number;
|
||||
@@ -167,13 +170,17 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
}
|
||||
}
|
||||
|
||||
// The administrator's own rules: judged only for servers that are reporting (one that is not has its offline alert).
|
||||
// The administrator's own rules, and what is unusual for the time: judged only for servers that are reporting (one that is not has its offline alert).
|
||||
if (!opts.skipServers) {
|
||||
const offlineIds = new Set(out.map((c) => /^offline:server:(\d+)$/.exec(c.key)?.[1]).filter((id): id is string => id !== undefined).map(Number));
|
||||
for (const f of snapshot.rules ?? []) {
|
||||
if (offlineIds.has(f.serverId)) continue;
|
||||
out.push({ key: findingKey(f), source: "server:rules", message: f.message, severity: f.severity });
|
||||
}
|
||||
for (const f of snapshot.baselines ?? []) {
|
||||
if (offlineIds.has(f.serverId)) continue;
|
||||
out.push({ key: baselineKey(f), source: "server:baseline", message: f.message, severity: "warning" });
|
||||
}
|
||||
}
|
||||
|
||||
for (const u of snapshot.unifi ?? []) {
|
||||
@@ -385,6 +392,16 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
held.add("server:rules");
|
||||
}
|
||||
|
||||
// What is unusually high for this hour of the week. A failure here costs only this check.
|
||||
if (checks.baselineAlerts) {
|
||||
try {
|
||||
snapshot.baselines = await computeBaselineFindings((await getSettings()).notifications.timezone || "UTC");
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge what is usual:", err instanceof Error ? err.message : err);
|
||||
held.add("server:baseline");
|
||||
}
|
||||
}
|
||||
|
||||
const rows = await db
|
||||
.select({ id: integrations.id, name: integrations.name, type: integrations.type })
|
||||
.from(integrations)
|
||||
|
||||
@@ -27,6 +27,8 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^rule:\d+:server:(\d+)(?::|$)/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^baseline:(?:cpu|mem):server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^offline:device:(\d+)$/.exec(key);
|
||||
|
||||
@@ -121,6 +121,8 @@ export interface HealthCheckSettings {
|
||||
deviceOfflineMinutes: number;
|
||||
/** Days before the end of an operating system's support that a server is first flagged on the Alerts page. 0 turns it off. */
|
||||
osSupportWarnDays: number;
|
||||
/** A server whose CPU load or memory use over the last hour is clearly higher than is normal for this hour of the week is a warning (services/baselines.ts). */
|
||||
baselineAlerts: boolean;
|
||||
}
|
||||
|
||||
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
|
||||
@@ -298,7 +300,7 @@ const DEFAULTS: AppSettings = {
|
||||
selfBackup: { enabled: true, time: "03:30", keep: 14 },
|
||||
profilePictures: { gravatar: false, identityProvider: false },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90 },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
consistency: { excludedRanges: ["172.16.0.0/12"] },
|
||||
|
||||
Reference in new issue
Block a user