History: server load, memory and disks are also kept as hourly (four months) and daily (over two years) averages with lowest and highest; endpoint answers as hourly (four months) beside the daily figures. Charts ask for 24 hours to a year and are drawn from whichever is fine enough. A disk that two weeks call flat is looked at again over ninety days for the forecast. Built once from what is kept. One chart: TimeChart replaces the sparkline and the multi-line chart: hover and arrow-key readout, a lowest/highest band, gaps shown as gaps, the same ranges everywhere and the choice remembered. Used for server trends, comparing servers, a check's availability and answer times (Endpoints > History) and a server's availability. Read API: /api/v1 (servers, metrics, alerts, incidents, inventory, network, endpoints and their history, maintenance) with personal read-only tokens made under the user menu: shown once, hashed, 30/90/365 days or none, ten at most, removed with their owner. GET only. Documented in API.md. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
250 lines
15 KiB
TypeScript
250 lines
15 KiB
TypeScript
/**
|
|
* Longer history: server figures and endpoint answer times kept by the hour and the day beyond the days they are kept in full, and drawn from whichever is
|
|
* fine enough for the range asked for, up to a year.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { AddressInfo } from "node:net";
|
|
import type { Server } from "node:http";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-history-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const express = (await import("express")).default;
|
|
const { db } = await import("../src/db/client.js");
|
|
const { users, servers, serverMetrics, serverMetricsRollup, endpointChecks, endpointResults, endpointHourly, endpointDaily } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const metrics = await import("../src/services/serverMetrics.js");
|
|
const checks = await import("../src/services/endpointChecks.js");
|
|
const { serversRouter } = await import("../src/routes/servers.js");
|
|
const { endpointsRouter } = await import("../src/routes/endpoints.js");
|
|
const { eq, and } = await import("drizzle-orm");
|
|
|
|
await runMigrations();
|
|
|
|
const DAY = 24 * 3600_000;
|
|
const HOUR = 3600_000;
|
|
const NOW = new Date(Math.floor(Date.now() / HOUR) * HOUR + 20 * 60_000); // twenty minutes into an hour
|
|
const ago = (days: number, extraMs = 0) => new Date(NOW.getTime() - days * DAY + extraMs);
|
|
const system = (cpu: number, disk: number) => ({ cpu: { load_percent: cpu }, memory: { total_bytes: 1000, used_bytes: 500 }, disks: [{ mount: "/", size_bytes: 100, used_bytes: disk }] }) as never;
|
|
|
|
let serverId = 0;
|
|
let checkId = 0;
|
|
|
|
before(async () => {
|
|
await db.insert(users).values([{ oidcSub: "ada", name: "Ada", role: "admin" }]);
|
|
serverId = (await db.insert(servers).values({ name: "web", apiTokenHash: "w", apiTokenPrefix: "w", lastSeenAt: NOW.toISOString() }).returning({ id: servers.id }))[0].id;
|
|
checkId = (await db.insert(endpointChecks).values({ name: "Site", kind: "tcp", target: "127.0.0.1", port: 1 }).returning({ id: endpointChecks.id }))[0].id;
|
|
});
|
|
after(() => {
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows may still hold the database file
|
|
}
|
|
});
|
|
|
|
describe("the hourly and daily figures of a server", () => {
|
|
test("which figures answer which range: the samples while they reach, then the hours, then the days", () => {
|
|
assert.deepEqual([1, 7, 35, 36, 120, 121, 365].map(metrics.sourceFor), ["samples", "samples", "samples", "hourly", "hourly", "daily", "daily"]);
|
|
});
|
|
|
|
test("a sample goes into its hour and its day, and the next one in the same hour is added to it, with the lowest and highest kept", async () => {
|
|
const t = new Date(ago(0).getTime() - 10 * 60_000); // ten minutes into the hour
|
|
assert.equal(await metrics.recordServerMetrics(serverId, system(10, 50), t), true);
|
|
assert.equal(await metrics.recordServerMetrics(serverId, system(30, 52), new Date(t.getTime() + 11 * 60_000)), true);
|
|
const cpuHour = (await db.select().from(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.metric, "cpu"), eq(serverMetricsRollup.grain, "h"))))[0];
|
|
assert.deepEqual([cpuHour.n, cpuHour.sum, cpuHour.min, cpuHour.max, cpuHour.at], [2, 40, 10, 30, metrics.hourStart(t.getTime())]);
|
|
const cpuDay = (await db.select().from(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.metric, "cpu"), eq(serverMetricsRollup.grain, "d"))))[0];
|
|
assert.deepEqual([cpuDay.n, cpuDay.at], [2, metrics.dayStart(t.getTime())]);
|
|
});
|
|
|
|
test("a hundred days of a disk filling: seven days come from the samples, ninety from the hours, a year from the days", async () => {
|
|
// one sample a day, climbing 0.3 points a day, oldest first (the way agents report)
|
|
const hist = (await db.insert(servers).values({ name: "hist", apiTokenHash: "h", apiTokenPrefix: "h", lastSeenAt: NOW.toISOString() }).returning({ id: servers.id }))[0].id;
|
|
for (let d = 100; d >= 1; d--) await metrics.recordServerMetrics(hist, system(20, 30 + (100 - d) * 0.3), ago(d));
|
|
const week = await metrics.getServerMetrics(hist, 7, NOW);
|
|
const disk7 = week.find((s) => s.metric === "disk:/")!;
|
|
assert.ok(disk7.points.length > 0 && disk7.points.every((p) => p.lo === undefined), "from the samples: no band");
|
|
|
|
const ninety = await metrics.getServerMetrics(hist, 90, NOW);
|
|
const disk90 = ninety.find((s) => s.metric === "disk:/")!;
|
|
assert.ok(disk90.points.length >= 85, `about a point a day (got ${disk90.points.length})`);
|
|
assert.ok(disk90.points.every((p) => p.lo !== undefined && p.hi !== undefined && p.lo <= p.v && p.v <= p.hi), "from the hours: each has a lowest and highest");
|
|
assert.ok(disk90.trend && Math.abs(disk90.trend.perDay - 0.3) < 0.05, "and the trend reaches back all ninety days");
|
|
assert.ok(disk90.trend!.spanDays > 85);
|
|
|
|
const year = await metrics.getServerMetrics(hist, 365, NOW);
|
|
const diskYear = year.find((s) => s.metric === "disk:/")!;
|
|
assert.ok(diskYear.points.length >= 99 && diskYear.points.length <= 370);
|
|
assert.equal(diskYear.latest, 30 + 99 * 0.3 > 100 ? 100 : Math.round((30 + 99 * 0.3) * 10) / 10, "the latest figure is the latest sample, not an average");
|
|
});
|
|
|
|
test("a range is never longer than a year, and never shorter than a day", async () => {
|
|
const huge = await metrics.getServerMetrics(serverId, 9999, NOW);
|
|
assert.ok(huge.length > 0);
|
|
const tiny = await metrics.getServerMetrics(serverId, 0, NOW);
|
|
assert.ok(Array.isArray(tiny));
|
|
});
|
|
|
|
test("a slow, steady climb that two weeks call flat is seen over ninety days", async () => {
|
|
const other = (await db.insert(servers).values({ name: "slow", apiTokenHash: "s", apiTokenPrefix: "s", lastSeenAt: NOW.toISOString() }).returning({ id: servers.id }))[0].id;
|
|
// 0.07 a day is under what two weeks of data round to being worth a forecast once the noise is in, but over ninety days it is plainly a climb
|
|
for (let d = 90; d >= 1; d--) await metrics.recordServerMetrics(other, system(10, 40 + (90 - d) * 0.07 + (d % 2 ? 0.01 : -0.01)), ago(d));
|
|
const series = await metrics.getForecastSeries(other, NOW);
|
|
const disk = series.find((s) => s.metric === "disk:/")!;
|
|
assert.ok(disk.trend && disk.trend.daysToFull !== null, "there is a forecast");
|
|
assert.ok(disk.trend!.fit > 0.9);
|
|
});
|
|
|
|
test("the hourly and daily figures are kept so long and no longer", async () => {
|
|
const old = (await db.insert(servers).values({ name: "old", apiTokenHash: "o", apiTokenPrefix: "o" }).returning({ id: servers.id }))[0].id;
|
|
await db.insert(serverMetricsRollup).values([
|
|
{ serverId: old, metric: "cpu", grain: "h", at: ago(metrics.HOURLY_KEEP_DAYS + 3).toISOString(), sum: 1, n: 1, min: 1, max: 1 },
|
|
{ serverId: old, metric: "cpu", grain: "d", at: ago(metrics.HOURLY_KEEP_DAYS + 3).toISOString(), sum: 1, n: 1, min: 1, max: 1 },
|
|
{ serverId: old, metric: "cpu", grain: "d", at: ago(metrics.DAILY_KEEP_DAYS + 3).toISOString(), sum: 1, n: 1, min: 1, max: 1 },
|
|
]);
|
|
await metrics.recordServerMetrics(old, system(5, 5), NOW);
|
|
const left = await db.select().from(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, old), eq(serverMetricsRollup.metric, "cpu")));
|
|
assert.deepEqual(left.map((r) => r.grain).sort(), ["d", "d", "h"], "the old hour is gone, the old day within two years stays, the one beyond is gone, and today's two are there");
|
|
assert.equal(left.filter((r) => r.grain === "h").length, 1);
|
|
});
|
|
|
|
test("the first time, the hours and days are built from the samples already kept — once, and exactly", async () => {
|
|
const legacy = (await db.insert(servers).values({ name: "legacy", apiTokenHash: "l", apiTokenPrefix: "l" }).returning({ id: servers.id }))[0].id;
|
|
const base = new Date("2026-03-10T10:05:00.000Z").getTime();
|
|
await db.insert(serverMetrics).values([
|
|
{ serverId: legacy, metric: "cpu", value: 10, recordedAt: new Date(base).toISOString() },
|
|
{ serverId: legacy, metric: "cpu", value: 20, recordedAt: new Date(base + 10 * 60_000).toISOString() },
|
|
{ serverId: legacy, metric: "cpu", value: 60, recordedAt: new Date(base + HOUR).toISOString() },
|
|
]);
|
|
assert.equal(await metrics.backfillMetricRollups(), true);
|
|
assert.equal(await metrics.backfillMetricRollups(), false, "done once");
|
|
const rows = await db.select().from(serverMetricsRollup).where(eq(serverMetricsRollup.serverId, legacy));
|
|
const hours = rows.filter((r) => r.grain === "h").sort((a, b) => a.at.localeCompare(b.at));
|
|
assert.deepEqual(hours.map((r) => [r.at, r.n, r.sum, r.min, r.max]), [["2026-03-10T10:00:00.000Z", 2, 30, 10, 20], ["2026-03-10T11:00:00.000Z", 1, 60, 60, 60]]);
|
|
const days = rows.filter((r) => r.grain === "d");
|
|
assert.deepEqual(days.map((r) => [r.at, r.n, r.sum, r.min, r.max]), [["2026-03-10T00:00:00.000Z", 3, 90, 10, 60]]);
|
|
});
|
|
});
|
|
|
|
describe("how a check has done over a long time", () => {
|
|
const probe = (ok: boolean, ms: number | null) => async () => ({ ok, ms, message: ok ? "open" : "refused" });
|
|
const row = () => db.select().from(endpointChecks).where(eq(endpointChecks.id, checkId)).then((r) => r[0]);
|
|
|
|
test("each answer goes into its hour, and the average, the slowest and how many answered are kept", async () => {
|
|
const t = ago(5, -45 * 60_000); // five days ago, a little into an hour
|
|
await checks.runCheck(await row(), t, probe(true, 100));
|
|
await checks.runCheck(await row(), new Date(t.getTime() + 60_000), probe(true, 300));
|
|
await checks.runCheck(await row(), new Date(t.getTime() + 120_000), probe(false, null));
|
|
const hour = (await db.select().from(endpointHourly).where(eq(endpointHourly.checkId, checkId)))[0];
|
|
assert.deepEqual([hour.total, hour.ok, hour.msSum, hour.msCount, hour.msMax], [3, 2, 400, 2, 300]);
|
|
});
|
|
|
|
test("bucketing adds the figures up, so a slice is as true as the checks in it", () => {
|
|
const rows = [
|
|
{ t: 0, total: 10, ok: 10, msSum: 1000, msCount: 10, msMax: 400 },
|
|
{ t: 1000, total: 10, ok: 5, msSum: 1000, msCount: 5, msMax: 900 },
|
|
{ t: 9000, total: 4, ok: 4, msSum: 400, msCount: 4, msMax: 100 },
|
|
];
|
|
const points = checks.bucketHistory(rows, 0, 10_000, 2);
|
|
assert.deepEqual(points.map((p) => [p.availability, p.avgMs, p.maxMs, p.checks]), [[75, 133, 900, 20], [100, 100, 100, 4]]);
|
|
assert.deepEqual(checks.bucketHistory([], 0, 10, 2), []);
|
|
assert.equal(checks.bucketHistory([{ t: 0, total: 1, ok: 0, msSum: 0, msCount: 0, msMax: 0 }], 0, 10, 1)[0].avgMs, null, "no timed answer, no average");
|
|
});
|
|
|
|
test("a day is drawn from the results, a month from the hours, and half a year from the days", async () => {
|
|
// some results of today, kept in full
|
|
await db.insert(endpointResults).values([
|
|
{ checkId, at: ago(0, -30 * 60_000).toISOString(), ok: true, ms: 80, message: "open" },
|
|
{ checkId, at: ago(0, -20 * 60_000).toISOString(), ok: false, ms: null, message: "refused" },
|
|
]);
|
|
// and daily figures for the last 200 days
|
|
for (let d = 200; d >= 7; d--) {
|
|
const day = ago(d).toISOString().slice(0, 10);
|
|
await db.insert(endpointDaily).values({ checkId, day, total: 288, ok: d % 50 === 0 ? 250 : 288, msSum: 288 * 40, msCount: 288 }).onConflictDoNothing();
|
|
}
|
|
const day = await checks.getEndpointHistory(checkId, 1, NOW);
|
|
assert.equal(day.source, "results");
|
|
assert.deepEqual([day.checks, day.availability, day.avgMs], [2, 50, 80]);
|
|
|
|
const month = await checks.getEndpointHistory(checkId, 30, NOW);
|
|
assert.equal(month.source, "hourly");
|
|
assert.equal(month.checks, 3, "the three asked five days ago");
|
|
assert.equal(month.availability, 66.7);
|
|
assert.equal(month.points[0].maxMs, 300);
|
|
|
|
const half = await checks.getEndpointHistory(checkId, 200, NOW);
|
|
assert.equal(half.source, "daily");
|
|
assert.ok(half.points.length >= 150 && half.points.length <= 200, "a point a day");
|
|
assert.ok(half.availability !== null && half.availability < 100 && half.availability > 98);
|
|
assert.equal(half.avgMs, 40);
|
|
});
|
|
|
|
test("the first time, the hours are built from the results still kept — once", async () => {
|
|
const fresh = (await db.insert(endpointChecks).values({ name: "Other", kind: "tcp", target: "127.0.0.1", port: 2 }).returning({ id: endpointChecks.id }))[0].id;
|
|
await db.insert(endpointResults).values([
|
|
{ checkId: fresh, at: "2026-03-10T10:05:00.000Z", ok: true, ms: 50, message: "" },
|
|
{ checkId: fresh, at: "2026-03-10T10:35:00.000Z", ok: true, ms: 150, message: "" },
|
|
{ checkId: fresh, at: "2026-03-10T10:45:00.000Z", ok: false, ms: null, message: "" },
|
|
]);
|
|
assert.equal(await checks.backfillEndpointHourly(), true);
|
|
assert.equal(await checks.backfillEndpointHourly(), false);
|
|
const [hour] = await db.select().from(endpointHourly).where(eq(endpointHourly.checkId, fresh));
|
|
assert.deepEqual([hour.hour, hour.total, hour.ok, hour.msSum, hour.msCount, hour.msMax], ["2026-03-10T10:00:00.000Z", 3, 2, 200, 2, 150]);
|
|
});
|
|
|
|
test("the hours are pruned after four months", async () => {
|
|
await db.insert(endpointHourly).values({ checkId, hour: ago(checks.KEEP_HOURLY_DAYS + 5).toISOString(), total: 1, ok: 1, msSum: 1, msCount: 1, msMax: 1 });
|
|
await checks.pruneResults(NOW);
|
|
const left = await db.select().from(endpointHourly).where(eq(endpointHourly.checkId, checkId));
|
|
assert.ok(left.every((r) => Date.parse(r.hour) > NOW.getTime() - checks.KEEP_HOURLY_DAYS * DAY));
|
|
});
|
|
});
|
|
|
|
describe("over the routes", () => {
|
|
let app: Server;
|
|
let base = "";
|
|
before(async () => {
|
|
const a = express();
|
|
a.use(express.json());
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: "ada" } };
|
|
next();
|
|
});
|
|
a.use("/api/servers", serversRouter);
|
|
a.use("/api/endpoints", endpointsRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
|
});
|
|
after(() => app.close());
|
|
|
|
test("a server's metrics can be asked for a year, and no more", async () => {
|
|
const year = (await (await fetch(`${base}/servers/${serverId}/metrics?days=365`)).json()) as any;
|
|
assert.equal(year.days, 365);
|
|
assert.ok(year.series.length > 0);
|
|
const more = (await (await fetch(`${base}/servers/${serverId}/metrics?days=5000`)).json()) as any;
|
|
assert.equal(more.days, 365);
|
|
const none = (await (await fetch(`${base}/servers/${serverId}/metrics`)).json()) as any;
|
|
assert.equal(none.days, 7);
|
|
});
|
|
|
|
test("a check's history: availability and answer times for a range, 404 for one that is not there", async () => {
|
|
const body = (await (await fetch(`${base}/endpoints/${checkId}/history?days=30`)).json()) as any;
|
|
assert.deepEqual([body.check.name, body.days, body.source, body.checks], ["Site", 30, "hourly", 5]); // the three asked five days ago, and the two results of today that the backfill above counted
|
|
assert.equal((await fetch(`${base}/endpoints/999999/history`)).status, 404);
|
|
assert.equal((await fetch(`${base}/endpoints/abc/history`)).status, 400);
|
|
assert.equal(((await (await fetch(`${base}/endpoints/${checkId}/history?days=99999`)).json()) as any).days, 365);
|
|
});
|
|
});
|