A year of history, one chart, and a read API with personal tokens
History: server load, memory and disks are also kept as hourly (four months) and daily (over two years) averages with lowest and highest; endpoint answers as hourly (four months) beside the daily figures. Charts ask for 24 hours to a year and are drawn from whichever is fine enough. A disk that two weeks call flat is looked at again over ninety days for the forecast. Built once from what is kept. One chart: TimeChart replaces the sparkline and the multi-line chart: hover and arrow-key readout, a lowest/highest band, gaps shown as gaps, the same ranges everywhere and the choice remembered. Used for server trends, comparing servers, a check's availability and answer times (Endpoints > History) and a server's availability. Read API: /api/v1 (servers, metrics, alerts, incidents, inventory, network, endpoints and their history, maintenance) with personal read-only tokens made under the user menu: shown once, hashed, 30/90/365 days or none, ten at most, removed with their owner. GET only. Documented in API.md. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5e26e77340
commit
68b41b5586
47 files changed
+12360
-247
No files matched your search
@@ -1,19 +1,30 @@
|
||||
/**
|
||||
* A short history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend
|
||||
* A history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend
|
||||
* (and say how soon a filling disk will run out) instead of only the latest figure.
|
||||
*
|
||||
* One sample is kept per server every ten minutes at most, however often its agent reports, for 35 days. Everything here is a
|
||||
* percentage: CPU load (as the agent reports it), memory in use, and each disk's used share.
|
||||
*
|
||||
* Beyond those 35 days the same figures are kept coarser: each sample also goes into its hour (kept four months) and its day (kept over two years), as a
|
||||
* sum, a count, a lowest and a highest. A chart asks for a number of days and is answered from whichever is fine enough and still there: the samples up to
|
||||
* 35 days, the hours up to 120, the days beyond. A year of a disk filling is then one point a day.
|
||||
*/
|
||||
import { and, desc, eq, gte, lt } from "drizzle-orm";
|
||||
import { and, desc, eq, gte, lt, sql } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { serverMetrics } from "../db/schema.js";
|
||||
import { serverMetrics, serverMetricsRollup } from "../db/schema.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
import type { IncomingSystemInfo } from "./taskSync.js";
|
||||
|
||||
/** Reports closer together than this add no sample. */
|
||||
export const MIN_GAP_MS = 10 * 60_000;
|
||||
export const RETENTION_DAYS = 35;
|
||||
/** How long the hourly and daily figures are kept. */
|
||||
export const HOURLY_KEEP_DAYS = 120;
|
||||
export const DAILY_KEEP_DAYS = 800;
|
||||
/** The longest stretch a chart may ask for. */
|
||||
export const MAX_RANGE_DAYS = 365;
|
||||
const DAY_MS = 24 * 3600_000;
|
||||
const HOUR_MS = 3600_000;
|
||||
|
||||
export interface Sample {
|
||||
metric: string;
|
||||
@@ -39,6 +50,35 @@ export function samplesFromSystem(system: IncomingSystemInfo | null | undefined)
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The start of the hour or day a moment falls in, as ISO. */
|
||||
export const hourStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / HOUR_MS) * HOUR_MS).toISOString();
|
||||
export const dayStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / DAY_MS) * DAY_MS).toISOString();
|
||||
|
||||
/** Adds samples to their hour and day. Never allowed to break recording the sample itself. */
|
||||
async function addToRollups(serverId: number, samples: Sample[], at: Date): Promise<void> {
|
||||
try {
|
||||
for (const grain of ["h", "d"] as const) {
|
||||
const bucket = grain === "h" ? hourStart(at.getTime()) : dayStart(at.getTime());
|
||||
for (const s of samples) {
|
||||
await db
|
||||
.insert(serverMetricsRollup)
|
||||
.values({ serverId, metric: s.metric, grain, at: bucket, sum: s.value, n: 1, min: s.value, max: s.value })
|
||||
.onConflictDoUpdate({
|
||||
target: [serverMetricsRollup.serverId, serverMetricsRollup.metric, serverMetricsRollup.grain, serverMetricsRollup.at],
|
||||
set: {
|
||||
sum: sql`${serverMetricsRollup.sum} + ${s.value}`,
|
||||
n: sql`${serverMetricsRollup.n} + 1`,
|
||||
min: sql`min(${serverMetricsRollup.min}, ${s.value})`,
|
||||
max: sql`max(${serverMetricsRollup.max}, ${s.value})`,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[metrics] couldn't add to the hourly and daily figures:", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Records a report's figures, unless this server was sampled a moment ago. Returns whether anything was stored. */
|
||||
export async function recordServerMetrics(serverId: number, system: IncomingSystemInfo | null | undefined, now: Date = new Date()): Promise<boolean> {
|
||||
const samples = samplesFromSystem(system);
|
||||
@@ -54,7 +94,32 @@ export async function recordServerMetrics(serverId: number, system: IncomingSyst
|
||||
|
||||
const at = now.toISOString();
|
||||
await db.insert(serverMetrics).values(samples.map((s) => ({ serverId, metric: s.metric, value: s.value, recordedAt: at })));
|
||||
await addToRollups(serverId, samples, now);
|
||||
await db.delete(serverMetrics).where(and(eq(serverMetrics.serverId, serverId), lt(serverMetrics.recordedAt, new Date(now.getTime() - RETENTION_DAYS * DAY_MS).toISOString())));
|
||||
await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "h"), lt(serverMetricsRollup.at, new Date(now.getTime() - HOURLY_KEEP_DAYS * DAY_MS).toISOString())));
|
||||
await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "d"), lt(serverMetricsRollup.at, new Date(now.getTime() - DAILY_KEEP_DAYS * DAY_MS).toISOString())));
|
||||
return true;
|
||||
}
|
||||
|
||||
const BACKFILL_FLAG = "metricsRollupBackfilled";
|
||||
|
||||
/**
|
||||
* The first time after the hourly and daily figures were added, builds them from the samples already kept (up to 35 days), so the longer ranges are not empty
|
||||
* for weeks. A bucket is written from what the samples say, replacing what is there, so it is right however far it got before. Done once.
|
||||
*/
|
||||
export async function backfillMetricRollups(): Promise<boolean> {
|
||||
if (await getInternalFlag(BACKFILL_FLAG)) return false;
|
||||
for (const [grain, len] of [["h", 13], ["d", 10]] as const) {
|
||||
const suffix = grain === "h" ? ":00:00.000Z" : "T00:00:00.000Z";
|
||||
await db.run(sql`
|
||||
INSERT INTO server_metrics_rollup (server_id, metric, grain, at, sum, n, min, max)
|
||||
SELECT server_id, metric, ${grain}, substr(recorded_at, 1, ${len}) || ${suffix}, sum(value), count(*), min(value), max(value)
|
||||
FROM server_metrics
|
||||
GROUP BY server_id, metric, substr(recorded_at, 1, ${len})
|
||||
ON CONFLICT (server_id, metric, grain, at) DO UPDATE SET sum = excluded.sum, n = excluded.n, min = excluded.min, max = excluded.max
|
||||
`);
|
||||
}
|
||||
await setInternalFlag(BACKFILL_FLAG, new Date().toISOString());
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -62,6 +127,9 @@ export interface Point {
|
||||
/** ISO time. */
|
||||
t: string;
|
||||
v: number;
|
||||
/** The lowest and highest figure in what this point stands for, when it stands for several (hourly and daily figures). */
|
||||
lo?: number;
|
||||
hi?: number;
|
||||
}
|
||||
|
||||
/** Averages points into at most `maxPoints` equal slices of [from, to], so a month of samples draws as a short line. */
|
||||
@@ -83,6 +151,42 @@ export function bucketPoints(points: { t: number; v: number }[], from: number, t
|
||||
.map(([, b]) => ({ t: new Date(b.tSum / b.n).toISOString(), v: round1(b.sum / b.n) }));
|
||||
}
|
||||
|
||||
/** A row of the hourly or daily figures, with the moment it starts at as epoch ms. */
|
||||
export interface RollupRow {
|
||||
t: number;
|
||||
sum: number;
|
||||
n: number;
|
||||
min: number;
|
||||
max: number;
|
||||
}
|
||||
|
||||
/** The average of what each row stands for. */
|
||||
export function rollupAverages(rows: RollupRow[]): { t: number; v: number }[] {
|
||||
return rows.filter((r) => r.n > 0).map((r) => ({ t: r.t, v: r.sum / r.n }));
|
||||
}
|
||||
|
||||
/** Rows into at most `maxPoints` slices of [from, to]: the average of the samples in each (not of the rows), with the lowest and highest. */
|
||||
export function bucketRollups(rows: RollupRow[], from: number, to: number, maxPoints: number): Point[] {
|
||||
if (rows.length === 0 || to <= from) return [];
|
||||
const size = Math.max((to - from) / maxPoints, 1);
|
||||
const buckets = new Map<number, { sum: number; n: number; lo: number; hi: number; tSum: number; rows: number }>();
|
||||
for (const r of rows) {
|
||||
if (r.t < from || r.t > to || r.n <= 0) continue;
|
||||
const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size));
|
||||
const b = buckets.get(key) ?? { sum: 0, n: 0, lo: Infinity, hi: -Infinity, tSum: 0, rows: 0 };
|
||||
b.sum += r.sum;
|
||||
b.n += r.n;
|
||||
b.lo = Math.min(b.lo, r.min);
|
||||
b.hi = Math.max(b.hi, r.max);
|
||||
b.tSum += r.t;
|
||||
b.rows += 1;
|
||||
buckets.set(key, b);
|
||||
}
|
||||
return [...buckets.entries()]
|
||||
.sort((a, b) => a[0] - b[0])
|
||||
.map(([, b]) => ({ t: new Date(b.tSum / b.rows).toISOString(), v: round1(b.sum / b.n), lo: round1(b.lo), hi: round1(b.hi) }));
|
||||
}
|
||||
|
||||
export interface Trend {
|
||||
/** Change in the percentage per day, from a straight line through the points. */
|
||||
perDay: number;
|
||||
@@ -175,35 +279,86 @@ function describe(metric: string): { kind: MetricSeries["kind"]; label: string }
|
||||
|
||||
const ORDER = { cpu: 0, mem: 1, disk: 2 } as const;
|
||||
|
||||
/** Every series recorded for a server over the last `days`, ready to draw. */
|
||||
/** Which of the kept figures answers a stretch of `days`: the samples while they reach back that far, then the hours, then the days. */
|
||||
export function sourceFor(days: number): "samples" | "hourly" | "daily" {
|
||||
return days <= RETENTION_DAYS ? "samples" : days <= HOURLY_KEEP_DAYS ? "hourly" : "daily";
|
||||
}
|
||||
|
||||
/** How many points a chart over `days` is drawn from at most. */
|
||||
const pointsFor = (days: number) => (days <= RETENTION_DAYS ? 96 : days <= HOURLY_KEEP_DAYS ? 200 : MAX_RANGE_DAYS + 5);
|
||||
|
||||
/** The latest sample of each figure of a server, whatever range is asked for. */
|
||||
async function latestSamples(serverId: number): Promise<Map<string, number>> {
|
||||
const rows = await db.all<{ metric: string; value: number }>(
|
||||
sql`SELECT metric, value FROM server_metrics WHERE id IN (SELECT max(id) FROM server_metrics WHERE server_id = ${serverId} GROUP BY metric)`,
|
||||
);
|
||||
return new Map(rows.map((r) => [r.metric, r.value]));
|
||||
}
|
||||
|
||||
/** Every series recorded for a server over the last `days` (up to a year), ready to draw. */
|
||||
export async function getServerMetrics(serverId: number, days: number, now: Date = new Date()): Promise<MetricSeries[]> {
|
||||
days = Math.min(MAX_RANGE_DAYS, Math.max(1, days));
|
||||
const to = now.getTime();
|
||||
const from = to - days * DAY_MS;
|
||||
const rows = await db
|
||||
.select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt })
|
||||
.from(serverMetrics)
|
||||
.where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString())))
|
||||
.orderBy(serverMetrics.recordedAt);
|
||||
const source = sourceFor(days);
|
||||
|
||||
const byMetric = new Map<string, { t: number; v: number }[]>();
|
||||
for (const r of rows) {
|
||||
const list = byMetric.get(r.metric) ?? [];
|
||||
list.push({ t: Date.parse(r.at), v: r.value });
|
||||
byMetric.set(r.metric, list);
|
||||
// The figures, per metric, as { t, v } for a trend and as points for the chart.
|
||||
const raw = new Map<string, { t: number; v: number }[]>();
|
||||
const points = new Map<string, Point[]>();
|
||||
if (source === "samples") {
|
||||
const rows = await db
|
||||
.select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt })
|
||||
.from(serverMetrics)
|
||||
.where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString())))
|
||||
.orderBy(serverMetrics.recordedAt);
|
||||
for (const r of rows) raw.set(r.metric, [...(raw.get(r.metric) ?? []), { t: Date.parse(r.at), v: r.value }]);
|
||||
for (const [metric, list] of raw) points.set(metric, bucketPoints(list, from, to, pointsFor(days)));
|
||||
} else {
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(serverMetricsRollup)
|
||||
.where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, source === "hourly" ? "h" : "d"), gte(serverMetricsRollup.at, new Date(from).toISOString())))
|
||||
.orderBy(serverMetricsRollup.at);
|
||||
const byMetric = new Map<string, RollupRow[]>();
|
||||
for (const r of rows) byMetric.set(r.metric, [...(byMetric.get(r.metric) ?? []), { t: Date.parse(r.at), sum: r.sum, n: r.n, min: r.min, max: r.max }]);
|
||||
for (const [metric, list] of byMetric) {
|
||||
raw.set(metric, rollupAverages(list));
|
||||
points.set(metric, bucketRollups(list, from, to, pointsFor(days)));
|
||||
}
|
||||
}
|
||||
const latest = source === "samples" ? null : await latestSamples(serverId);
|
||||
|
||||
return [...byMetric.entries()]
|
||||
.map(([metric, raw]) => {
|
||||
return [...raw.entries()]
|
||||
.map(([metric, list]) => {
|
||||
const { kind, label } = describe(metric);
|
||||
const newest = latest?.get(metric) ?? list[list.length - 1]?.v;
|
||||
return {
|
||||
metric,
|
||||
kind,
|
||||
label,
|
||||
points: bucketPoints(raw, from, to, 96),
|
||||
latest: raw[raw.length - 1]?.v ?? null,
|
||||
points: points.get(metric) ?? [],
|
||||
latest: newest === undefined ? null : source === "samples" ? newest : round1(newest),
|
||||
// A disk filling is the thing worth forecasting; load and memory go up and down by the hour.
|
||||
trend: kind === "disk" ? trendOf(raw) : null,
|
||||
trend: kind === "disk" ? trendOf(list) : null,
|
||||
};
|
||||
})
|
||||
.sort((a, b) => ORDER[a.kind] - ORDER[b.kind] || a.label.localeCompare(b.label));
|
||||
}
|
||||
|
||||
/** How far back the longer look at a disk goes when the last two weeks say it is flat. */
|
||||
export const LONG_FORECAST_DAYS = 90;
|
||||
|
||||
/**
|
||||
* The series to forecast from: the last two weeks, as before — except that a disk those two weeks call flat is looked at again over the last
|
||||
* ninety days, so a slow, steady climb (a point a week) is seen too. The longer trend replaces the short one only if it is really rising.
|
||||
*/
|
||||
export async function getForecastSeries(serverId: number, now: Date = new Date()): Promise<MetricSeries[]> {
|
||||
const short = await getServerMetrics(serverId, 14, now);
|
||||
if (!short.some((s) => s.kind === "disk" && (!s.trend || s.trend.daysToFull === null))) return short;
|
||||
const long = await getServerMetrics(serverId, LONG_FORECAST_DAYS, now);
|
||||
return short.map((s) => {
|
||||
if (s.kind !== "disk" || (s.trend && s.trend.daysToFull !== null)) return s;
|
||||
const longer = long.find((l) => l.metric === s.metric);
|
||||
return longer?.trend && longer.trend.daysToFull !== null ? { ...s, trend: longer.trend } : s;
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user