/** * A history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend * (and say how soon a filling disk will run out) instead of only the latest figure. * * One sample is kept per server every ten minutes at most, however often its agent reports, for 35 days. Everything here is a * percentage: CPU load (as the agent reports it), memory in use, and each disk's used share. * * Beyond those 35 days the same figures are kept coarser: each sample also goes into its hour (kept four months) and its day (kept over two years), as a * sum, a count, a lowest and a highest. A chart asks for a number of days and is answered from whichever is fine enough and still there: the samples up to * 35 days, the hours up to 120, the days beyond. A year of a disk filling is then one point a day. */ import { and, desc, eq, gte, lt, sql } from "drizzle-orm"; import { db } from "../db/client.js"; import { serverMetrics, serverMetricsRollup } from "../db/schema.js"; import { getInternalFlag, setInternalFlag } from "./settingsStore.js"; import type { IncomingSystemInfo } from "./taskSync.js"; /** Reports closer together than this add no sample. */ export const MIN_GAP_MS = 10 * 60_000; export const RETENTION_DAYS = 35; /** How long the hourly and daily figures are kept. */ export const HOURLY_KEEP_DAYS = 120; export const DAILY_KEEP_DAYS = 800; /** The longest stretch a chart may ask for. */ export const MAX_RANGE_DAYS = 365; const DAY_MS = 24 * 3600_000; const HOUR_MS = 3600_000; export interface Sample { metric: string; value: number; } const round1 = (n: number) => Math.round(n * 10) / 10; const clampPercent = (n: number) => Math.min(100, Math.max(0, n)); /** The percentages in one agent report. Anything missing or nonsensical (no size, not a number) is left out. */ export function samplesFromSystem(system: IncomingSystemInfo | null | undefined): Sample[] { if (!system) return []; const out: Sample[] = []; const load = system.cpu?.load_percent; if (typeof load === "number" && Number.isFinite(load)) out.push({ metric: "cpu", value: round1(clampPercent(load)) }); const total = system.memory?.total_bytes; const used = system.memory?.used_bytes; if (typeof total === "number" && typeof used === "number" && total > 0) out.push({ metric: "mem", value: round1(clampPercent((used / total) * 100)) }); for (const d of system.disks ?? []) { if (typeof d.mount !== "string" || !d.mount || !(d.size_bytes > 0) || typeof d.used_bytes !== "number") continue; out.push({ metric: `disk:${d.mount}`, value: round1(clampPercent((d.used_bytes / d.size_bytes) * 100)) }); } return out; } /** The start of the hour or day a moment falls in, as ISO. */ export const hourStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / HOUR_MS) * HOUR_MS).toISOString(); export const dayStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / DAY_MS) * DAY_MS).toISOString(); /** Adds samples to their hour and day. Never allowed to break recording the sample itself. */ async function addToRollups(serverId: number, samples: Sample[], at: Date): Promise { try { for (const grain of ["h", "d"] as const) { const bucket = grain === "h" ? hourStart(at.getTime()) : dayStart(at.getTime()); for (const s of samples) { await db .insert(serverMetricsRollup) .values({ serverId, metric: s.metric, grain, at: bucket, sum: s.value, n: 1, min: s.value, max: s.value }) .onConflictDoUpdate({ target: [serverMetricsRollup.serverId, serverMetricsRollup.metric, serverMetricsRollup.grain, serverMetricsRollup.at], set: { sum: sql`${serverMetricsRollup.sum} + ${s.value}`, n: sql`${serverMetricsRollup.n} + 1`, min: sql`min(${serverMetricsRollup.min}, ${s.value})`, max: sql`max(${serverMetricsRollup.max}, ${s.value})`, }, }); } } } catch (err) { console.error("[metrics] couldn't add to the hourly and daily figures:", err); } } /** Records a report's figures, unless this server was sampled a moment ago. Returns whether anything was stored. */ export async function recordServerMetrics(serverId: number, system: IncomingSystemInfo | null | undefined, now: Date = new Date()): Promise { const samples = samplesFromSystem(system); if (samples.length === 0) return false; const [latest] = await db .select({ at: serverMetrics.recordedAt }) .from(serverMetrics) .where(eq(serverMetrics.serverId, serverId)) .orderBy(desc(serverMetrics.recordedAt)) .limit(1); if (latest && now.getTime() - Date.parse(latest.at) < MIN_GAP_MS) return false; const at = now.toISOString(); await db.insert(serverMetrics).values(samples.map((s) => ({ serverId, metric: s.metric, value: s.value, recordedAt: at }))); await addToRollups(serverId, samples, now); await db.delete(serverMetrics).where(and(eq(serverMetrics.serverId, serverId), lt(serverMetrics.recordedAt, new Date(now.getTime() - RETENTION_DAYS * DAY_MS).toISOString()))); await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "h"), lt(serverMetricsRollup.at, new Date(now.getTime() - HOURLY_KEEP_DAYS * DAY_MS).toISOString()))); await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "d"), lt(serverMetricsRollup.at, new Date(now.getTime() - DAILY_KEEP_DAYS * DAY_MS).toISOString()))); return true; } const BACKFILL_FLAG = "metricsRollupBackfilled"; /** * The first time after the hourly and daily figures were added, builds them from the samples already kept (up to 35 days), so the longer ranges are not empty * for weeks. A bucket is written from what the samples say, replacing what is there, so it is right however far it got before. Done once. */ export async function backfillMetricRollups(): Promise { if (await getInternalFlag(BACKFILL_FLAG)) return false; for (const [grain, len] of [["h", 13], ["d", 10]] as const) { const suffix = grain === "h" ? ":00:00.000Z" : "T00:00:00.000Z"; await db.run(sql` INSERT INTO server_metrics_rollup (server_id, metric, grain, at, sum, n, min, max) SELECT server_id, metric, ${grain}, substr(recorded_at, 1, ${len}) || ${suffix}, sum(value), count(*), min(value), max(value) FROM server_metrics GROUP BY server_id, metric, substr(recorded_at, 1, ${len}) ON CONFLICT (server_id, metric, grain, at) DO UPDATE SET sum = excluded.sum, n = excluded.n, min = excluded.min, max = excluded.max `); } await setInternalFlag(BACKFILL_FLAG, new Date().toISOString()); return true; } export interface Point { /** ISO time. */ t: string; v: number; /** The lowest and highest figure in what this point stands for, when it stands for several (hourly and daily figures). */ lo?: number; hi?: number; } /** Averages points into at most `maxPoints` equal slices of [from, to], so a month of samples draws as a short line. */ export function bucketPoints(points: { t: number; v: number }[], from: number, to: number, maxPoints: number): Point[] { if (points.length === 0 || to <= from) return []; const size = Math.max((to - from) / maxPoints, 1); const buckets = new Map(); for (const p of points) { if (p.t < from || p.t > to) continue; const key = Math.min(maxPoints - 1, Math.floor((p.t - from) / size)); const b = buckets.get(key) ?? { sum: 0, n: 0, tSum: 0 }; b.sum += p.v; b.n += 1; b.tSum += p.t; buckets.set(key, b); } return [...buckets.entries()] .sort((a, b) => a[0] - b[0]) .map(([, b]) => ({ t: new Date(b.tSum / b.n).toISOString(), v: round1(b.sum / b.n) })); } /** A row of the hourly or daily figures, with the moment it starts at as epoch ms. */ export interface RollupRow { t: number; sum: number; n: number; min: number; max: number; } /** The average of what each row stands for. */ export function rollupAverages(rows: RollupRow[]): { t: number; v: number }[] { return rows.filter((r) => r.n > 0).map((r) => ({ t: r.t, v: r.sum / r.n })); } /** Rows into at most `maxPoints` slices of [from, to]: the average of the samples in each (not of the rows), with the lowest and highest. */ export function bucketRollups(rows: RollupRow[], from: number, to: number, maxPoints: number): Point[] { if (rows.length === 0 || to <= from) return []; const size = Math.max((to - from) / maxPoints, 1); const buckets = new Map(); for (const r of rows) { if (r.t < from || r.t > to || r.n <= 0) continue; const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size)); const b = buckets.get(key) ?? { sum: 0, n: 0, lo: Infinity, hi: -Infinity, tSum: 0, rows: 0 }; b.sum += r.sum; b.n += r.n; b.lo = Math.min(b.lo, r.min); b.hi = Math.max(b.hi, r.max); b.tSum += r.t; b.rows += 1; buckets.set(key, b); } return [...buckets.entries()] .sort((a, b) => a[0] - b[0]) .map(([, b]) => ({ t: new Date(b.tSum / b.rows).toISOString(), v: round1(b.sum / b.n), lo: round1(b.lo), hi: round1(b.hi) })); } export interface Trend { /** Change in the percentage per day, from a straight line through the points. */ perDay: number; /** How well a straight line describes the points, 0 to 1 (R²). A steady climb is near 1; a single jump or a wobble is low. */ fit: number; /** How much time the points cover, in days. */ spanDays: number; /** For something that fills up: days until it reaches 100% at that rate. Null when it isn't rising or the rate is negligible. */ daysToFull: number | null; } /** * A straight-line fit over the points. Needs at least six points covering at least a day; with less there is no honest trend to * give. A slope under 0.05 points a day counts as flat, so a disk that wobbles by a fraction isn't forecast to fill in a century. */ export function trendOf(points: { t: number; v: number }[]): Trend | null { if (points.length < 6) return null; const span = points[points.length - 1].t - points[0].t; if (span < DAY_MS) return null; const t0 = points[0].t; const xs = points.map((p) => (p.t - t0) / DAY_MS); const ys = points.map((p) => p.v); const n = points.length; const meanX = xs.reduce((a, b) => a + b, 0) / n; const meanY = ys.reduce((a, b) => a + b, 0) / n; let num = 0; let den = 0; let totalVariance = 0; for (let i = 0; i < n; i++) { num += (xs[i] - meanX) * (ys[i] - meanY); den += (xs[i] - meanX) ** 2; totalVariance += (ys[i] - meanY) ** 2; } if (den === 0) return null; const slope = num / den; // R²: the share of the movement the straight line explains. A flat series has nothing to explain, so it is no "fit" at all. const fit = totalVariance === 0 ? 0 : Math.round(((num * num) / (den * totalVariance)) * 100) / 100; const perDay = Math.round(slope * 100) / 100; const last = ys[n - 1]; const daysToFull = perDay >= 0.05 && last < 100 ? Math.ceil((100 - last) / perDay) : null; return { perDay, fit, spanDays: Math.round((span / DAY_MS) * 10) / 10, daysToFull }; } export interface DiskForecast { mount: string; latest: number; perDay: number; daysToFull: number; } /** What a forecast needs before it is worth raising an alert about: enough history, and growth that really is steady. */ export const FORECAST_MIN_SPAN_DAYS = 3; export const FORECAST_MIN_FIT = 0.9; /** * The disks that are heading for full within `withinDays`. A disk already at or above `alreadyAlertedAt` percent is left out — the * ordinary "disk nearly full" alert covers it — and so is one whose growth is erratic (a big copy, a log that rotates), where a * straight-line forecast would just be a guess. `alreadyAlertedAt` may be a function of the disk, for servers with a limit of their own * for one of them; returning null leaves that disk out altogether (it has been told not to alert). */ export function disksFillingSoon(series: MetricSeries[], withinDays: number, alreadyAlertedAt: number | ((mount: string) => number | null)): DiskForecast[] { const out: DiskForecast[] = []; for (const s of series) { if (s.kind !== "disk" || !s.trend || s.latest === null) continue; const { perDay, fit, spanDays, daysToFull } = s.trend; if (daysToFull === null || daysToFull > withinDays) continue; if (spanDays < FORECAST_MIN_SPAN_DAYS || fit < FORECAST_MIN_FIT) continue; const mount = s.metric.slice("disk:".length); const limit = typeof alreadyAlertedAt === "function" ? alreadyAlertedAt(mount) : alreadyAlertedAt; if (limit === null || s.latest >= limit) continue; out.push({ mount, latest: s.latest, perDay, daysToFull }); } return out.sort((a, b) => a.daysToFull - b.daysToFull); } export interface MetricSeries { metric: string; kind: "cpu" | "mem" | "disk"; label: string; points: Point[]; latest: number | null; trend: Trend | null; } function describe(metric: string): { kind: MetricSeries["kind"]; label: string } { if (metric === "cpu") return { kind: "cpu", label: "CPU load" }; if (metric === "mem") return { kind: "mem", label: "Memory" }; return { kind: "disk", label: `Disk ${metric.slice("disk:".length)}` }; } const ORDER = { cpu: 0, mem: 1, disk: 2 } as const; /** Which of the kept figures answers a stretch of `days`: the samples while they reach back that far, then the hours, then the days. */ export function sourceFor(days: number): "samples" | "hourly" | "daily" { return days <= RETENTION_DAYS ? "samples" : days <= HOURLY_KEEP_DAYS ? "hourly" : "daily"; } /** How many points a chart over `days` is drawn from at most. */ const pointsFor = (days: number) => (days <= RETENTION_DAYS ? 96 : days <= HOURLY_KEEP_DAYS ? 200 : MAX_RANGE_DAYS + 5); /** The latest sample of each figure of a server, whatever range is asked for. */ async function latestSamples(serverId: number): Promise> { const rows = await db.all<{ metric: string; value: number }>( sql`SELECT metric, value FROM server_metrics WHERE id IN (SELECT max(id) FROM server_metrics WHERE server_id = ${serverId} GROUP BY metric)`, ); return new Map(rows.map((r) => [r.metric, r.value])); } /** Every series recorded for a server over the last `days` (up to a year), ready to draw. */ export async function getServerMetrics(serverId: number, days: number, now: Date = new Date()): Promise { days = Math.min(MAX_RANGE_DAYS, Math.max(1, days)); const to = now.getTime(); const from = to - days * DAY_MS; const source = sourceFor(days); // The figures, per metric, as { t, v } for a trend and as points for the chart. const raw = new Map(); const points = new Map(); if (source === "samples") { const rows = await db .select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt }) .from(serverMetrics) .where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString()))) .orderBy(serverMetrics.recordedAt); for (const r of rows) raw.set(r.metric, [...(raw.get(r.metric) ?? []), { t: Date.parse(r.at), v: r.value }]); for (const [metric, list] of raw) points.set(metric, bucketPoints(list, from, to, pointsFor(days))); } else { const rows = await db .select() .from(serverMetricsRollup) .where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, source === "hourly" ? "h" : "d"), gte(serverMetricsRollup.at, new Date(from).toISOString()))) .orderBy(serverMetricsRollup.at); const byMetric = new Map(); for (const r of rows) byMetric.set(r.metric, [...(byMetric.get(r.metric) ?? []), { t: Date.parse(r.at), sum: r.sum, n: r.n, min: r.min, max: r.max }]); for (const [metric, list] of byMetric) { raw.set(metric, rollupAverages(list)); points.set(metric, bucketRollups(list, from, to, pointsFor(days))); } } const latest = source === "samples" ? null : await latestSamples(serverId); return [...raw.entries()] .map(([metric, list]) => { const { kind, label } = describe(metric); const newest = latest?.get(metric) ?? list[list.length - 1]?.v; return { metric, kind, label, points: points.get(metric) ?? [], latest: newest === undefined ? null : source === "samples" ? newest : round1(newest), // A disk filling is the thing worth forecasting; load and memory go up and down by the hour. trend: kind === "disk" ? trendOf(list) : null, }; }) .sort((a, b) => ORDER[a.kind] - ORDER[b.kind] || a.label.localeCompare(b.label)); } /** How far back the longer look at a disk goes when the last two weeks say it is flat. */ export const LONG_FORECAST_DAYS = 90; /** * The series to forecast from: the last two weeks, as before — except that a disk those two weeks call flat is looked at again over the last * ninety days, so a slow, steady climb (a point a week) is seen too. The longer trend replaces the short one only if it is really rising. */ export async function getForecastSeries(serverId: number, now: Date = new Date()): Promise { const short = await getServerMetrics(serverId, 14, now); if (!short.some((s) => s.kind === "disk" && (!s.trend || s.trend.daysToFull === null))) return short; const long = await getServerMetrics(serverId, LONG_FORECAST_DAYS, now); return short.map((s) => { if (s.kind !== "disk" || (s.trend && s.trend.daysToFull !== null)) return s; const longer = long.find((l) => l.metric === s.metric); return longer?.trend && longer.trend.daysToFull !== null ? { ...s, trend: longer.trend } : s; }); }