What is normal for the time: baselines per hour of the week

Each server's CPU load and memory use get a baseline for every hour of the week
(median and robust spread over up to ten weeks, four needed). A clearly high last
hour that never dropped back toward normal is a warning; everything else stays
quiet. Server History -> Trends draws a usual line and band. Setting under
Notifications, on by default.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-21 21:57:57 +02:00
1 parent 232a5661ab
commit 31044b2b64
17 files changed
+619 -10

No files matched your search

+19
View File
@@ -301,6 +301,22 @@ export interface ServerActivityEvent {
actor: string | null;
}
/** What is normal for one hour of the week: the median of that hour's averages over past weeks, a robust spread, and how many weeks it stands on. */
export interface BaselineSlot {
dow: number;
hour: number;
median: number;
spread: number;
weeks: number;
}
/** What is normal for a server at each hour of the week (Sunday 00:00 first), in the notification time zone; null where there are not enough weeks yet. */
export interface BaselineView {
timezone: string;
now: { dow: number; hour: number };
slots: { cpu: (BaselineSlot | null)[]; mem: (BaselineSlot | null)[] };
}
export interface ServerMetricSeries {
metric: string;
kind: "cpu" | "mem" | "disk";
@@ -519,6 +535,8 @@ export interface HealthCheckSettings {
deviceOfflineMinutes: number;
/** Days before an operating system's support ends that a server is flagged (0 = off). */
osSupportWarnDays: number;
/** A server whose CPU or memory over the last hour is clearly higher than is normal for this hour of the week is a warning. */
baselineAlerts: boolean;
}
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. */
@@ -2908,6 +2926,7 @@ export const api = {
detail: (id: number) => request<ServerDetail>(`/api/servers/${id}/detail`),
activity: (id: number) => request<{ events: ServerActivityEvent[] }>(`/api/servers/${id}/activity`),
metrics: (id: number, days: number) => request<{ days: number; series: ServerMetricSeries[] }>(`/api/servers/${id}/metrics?days=${days}`),
baseline: (id: number) => request<BaselineView>(`/api/servers/${id}/baseline`),
remove: (id: number) => request<void>(`/api/servers/${id}`, { method: "DELETE" }),
setTags: (id: number, tags: string[]) =>
request<{ tags: string[] }>(`/api/servers/${id}/tags`, { method: "PUT", body: JSON.stringify({ tags }) }),
+41 -4
View File
@@ -1,5 +1,6 @@
import { useEffect, useMemo, useState } from "react";
import { api, type ServerMetricSeries } from "../api/client";
import { api, type BaselineView, type ServerMetricSeries } from "../api/client";
import { hourOfWeek } from "../utils/hourOfWeek";
import TimeChart from "./TimeChart";
import RangeSelect from "./RangeSelect";
import { BlockSkeleton } from "./Skeleton";
@@ -33,6 +34,7 @@ export default function ServerTrends({ serverId }: { serverId: number }) {
const [days, setDays] = useChartRange(7);
const [series, setSeries] = useState<ServerMetricSeries[] | null>(null);
const [failed, setFailed] = useState(false);
const [baseline, setBaseline] = useState<BaselineView | null>(null);
// Fixed per load, so the right-hand end of a chart is when it was read and does not creep as the page re-renders.
const [to, setTo] = useState(() => Date.now());
@@ -52,11 +54,40 @@ export default function ServerTrends({ serverId }: { serverId: number }) {
};
}, [serverId, days]);
// What is usual for each hour of the week, loaded once; it only matters while the points are single samples or hours (the weekday means nothing for a day's average).
useEffect(() => {
let alive = true;
api.servers
.baseline(serverId)
.then((res) => alive && setBaseline(res))
.catch(() => alive && setBaseline(null));
return () => {
alive = false;
};
}, [serverId]);
const from = to - days * 86_400_000;
const rangeLabel = CHART_RANGES.find((r) => r.days === days)?.label ?? `${days} days`;
const charts = useMemo(
() => (series ?? []).map((s) => ({ s, words: describeTrend(s), series: [{ name: s.label, color: levelColor(s), points: s.points }] })),
[series],
() =>
(series ?? []).map((s) => {
const lines = [{ name: s.label, color: levelColor(s), points: s.points as { t: string; v: number; lo?: number; hi?: number }[] }];
const slots = s.kind === "disk" || !baseline ? null : baseline.slots[s.kind];
if (slots && days <= 30) {
// The usual for the hour each point falls in, as a line with a band around it: the median and two spreads either side.
const usual = s.points.flatMap((p) => {
const h = hourOfWeek(Date.parse(p.t), baseline!.timezone);
const slot = slots[h.dow * 24 + h.hour];
if (!slot) return [];
const room = 2 * Math.max(slot.spread, s.kind === "cpu" ? 4 : 2);
return [{ t: p.t, v: slot.median, lo: Math.max(0, slot.median - room), hi: Math.min(100, slot.median + room) }];
});
if (usual.length > 0) lines.push({ name: "Usual for the time", color: "#868e96", points: usual });
}
const here = s.kind === "disk" || !baseline ? null : baseline.slots[s.kind][baseline.now.dow * 24 + baseline.now.hour];
return { s, words: describeTrend(s), series: lines, usualNow: here };
}),
[series, baseline, days],
);
return (
@@ -76,16 +107,22 @@ export default function ServerTrends({ serverId }: { serverId: number }) {
</div>
) : (
<div className="trend-grid">
{charts.map(({ s, words, series: lines }) => (
{charts.map(({ s, words, series: lines, usualNow }) => (
<div key={s.metric} className="trend-block mb-3">
<div className="d-flex flex-wrap align-items-baseline gap-2 mb-1">
<strong>{s.label}</strong>
<span>{s.latest !== null ? `${Math.round(s.latest)}%` : "—"}</span>
<span className="text-secondary small">{words}</span>
{usualNow && (
<span className="text-secondary small" title={`The median of this hour of the week over ${usualNow.weeks} weeks`}>
· usually about {Math.round(usualNow.median)}% now
</span>
)}
</div>
<TimeChart series={lines} from={from} to={to} compact label={`${s.label} over the last ${rangeLabel}, ${s.latest !== null ? `${Math.round(s.latest)}% now` : "no reading"}`} />
</div>
))}
{charts.some((c) => c.series.length > 1) && <div className="text-secondary small">The grey band is what is usual for that hour of the week, from this server&rsquo;s own history (the median, two spreads either side).</div>}
{days > 35 && <div className="text-secondary small">Over {days > 120 ? "more than four months" : "more than five weeks"} each point is {days > 120 ? "a day" : "an hour"}&rsquo;s average, and the shaded band is its lowest and highest.</div>}
</div>
)}
@@ -83,7 +83,7 @@ const DIGEST_DAYS: { value: number; label: string }[] = [
{ value: 6, label: "Saturday" },
{ value: 0, label: "Sunday" },
];
const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90 };
const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true };
const DEFAULT_MONTHLY: MonthlyReportSettings = { enabled: false, day: 1, time: "08:00" };
type TestResult = { ok: boolean; message: string } | null;
@@ -754,6 +754,17 @@ export default function NotificationSettings() {
hint="Switches, access points and gateways read from a UniFi controller (Network devices). The controller is read every five minutes, and what a device needs or what needs it is folded under it when it is down."
onChange={(v) => setHealthChecks((h) => ({ ...h, deviceOfflineMinutes: v }))}
/>
<label className="form-check mt-3 mb-0">
<input type="checkbox" className="form-check-input" checked={healthChecks.baselineAlerts} onChange={(e) => setHealthChecks((h) => ({ ...h, baselineAlerts: e.target.checked }))} />
<span className="form-check-label">
Warn when CPU or memory is unusually high for the time
<span className="form-hint d-block">
Each server&rsquo;s CPU load and memory use are compared with what is normal for the same hour of the same weekday, from its own history (it needs four weeks of that hour, and uses up to ten; the
hours are those of the notification time zone below). It is quiet on purpose: only a <em>high</em> reading, last hour averaging at least 50&nbsp;% CPU or 70&nbsp;% memory, far above the usual and never back toward it, is
a warning. Sent with the server offline and disk messages.
</span>
</span>
</label>
<label className="form-check mt-3 mb-0">
<input type="checkbox" className="form-check-input" checked={healthChecks.newPortAlerts} onChange={(e) => setHealthChecks((h) => ({ ...h, newPortAlerts: e.target.checked }))} />
<span className="form-check-label">
+17
View File
@@ -0,0 +1,17 @@
const SHORT = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const formatters = new Map<string, Intl.DateTimeFormat>();
/** The weekday (0 = Sunday) and hour a moment falls in, in a time zone — the same hours of the week the server's baselines are made of. */
export function hourOfWeek(ms: number, timeZone: string): { dow: number; hour: number } {
let f = formatters.get(timeZone);
if (!f) {
try {
f = new Intl.DateTimeFormat("en-US", { timeZone, weekday: "short", hour: "2-digit", hourCycle: "h23" });
} catch {
f = new Intl.DateTimeFormat("en-US", { timeZone: "UTC", weekday: "short", hour: "2-digit", hourCycle: "h23" });
}
formatters.set(timeZone, f);
}
const parts = f.formatToParts(new Date(ms));
return { dow: Math.max(0, SHORT.indexOf(parts.find((p) => p.type === "weekday")?.value ?? "Sun")), hour: Number(parts.find((p) => p.type === "hour")?.value ?? 0) % 24 };
}
+1
View File
@@ -20,6 +20,7 @@ export const SETTINGS_INDEX: SettingsEntry[] = [
{ title: "Server offline after (minutes)", where: "Notifications › What to send › Servers and disks", to: `${N}?tab=events`, words: "offline stopped reporting silent agent threshold limit minutes" },
{ title: "Disk usage alert at (percent)", where: "Notifications › What to send › Servers and disks", to: `${N}?tab=events`, words: "disk full storage percent threshold space" },
{ title: "Warn when a disk will fill within (days)", where: "Notifications › What to send › Servers and disks", to: `${N}?tab=events`, words: "disk forecast fill growth trend days" },
{ title: "Warn when CPU or memory is unusually high for the time", where: "Notifications › What to send › Servers and disks", to: `${N}?tab=events`, words: "baseline usual normal unusual unusually busy high cpu memory load weekday hour history anomaly" },
{ title: "Warn when a server starts listening on a new port", where: "Notifications › What to send › Servers and disks", to: `${N}?tab=events`, words: "new port baseline listening tcp monitoring" },
{ title: "Call an alert flapping after (raises in an hour)", where: "Notifications › What to send › Servers and disks", to: `${N}?tab=events`, words: "flapping flap noise repeat raised cleared again hold back" },
{ title: "Alert when a network device is offline for (minutes)", where: "Notifications › What to send › Servers and disks", to: `${N}?tab=events`, words: "unifi network device switch access point gateway offline" },