History: server load, memory and disks are also kept as hourly (four months) and daily (over two years) averages with lowest and highest; endpoint answers as hourly (four months) beside the daily figures. Charts ask for 24 hours to a year and are drawn from whichever is fine enough. A disk that two weeks call flat is looked at again over ninety days for the forecast. Built once from what is kept. One chart: TimeChart replaces the sparkline and the multi-line chart: hover and arrow-key readout, a lowest/highest band, gaps shown as gaps, the same ranges everywhere and the choice remembered. Used for server trends, comparing servers, a check's availability and answer times (Endpoints > History) and a server's availability. Read API: /api/v1 (servers, metrics, alerts, incidents, inventory, network, endpoints and their history, maintenance) with personal read-only tokens made under the user menu: shown once, hashed, 30/90/365 days or none, ten at most, removed with their owner. GET only. Documented in API.md. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
203 lines
8.8 KiB
TypeScript
203 lines
8.8 KiB
TypeScript
/**
|
||
* The short history kept of each agent-reported server's load, memory and disks, and the trend drawn from it.
|
||
*
|
||
* Run with: npm test --workspace server
|
||
*/
|
||
import { test, describe, before, after } from "node:test";
|
||
import assert from "node:assert/strict";
|
||
import { mkdtempSync, rmSync } from "node:fs";
|
||
import { tmpdir } from "node:os";
|
||
import { join } from "node:path";
|
||
import type { AddressInfo } from "node:net";
|
||
import type { Server } from "node:http";
|
||
|
||
const dir = mkdtempSync(join(tmpdir(), "hm-metrics-"));
|
||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||
process.env.SESSION_DIR = join(dir, "sessions");
|
||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||
|
||
const express = (await import("express")).default;
|
||
const { eq } = await import("drizzle-orm");
|
||
const { db } = await import("../src/db/client.js");
|
||
const { users, servers, serverMetrics } = await import("../src/db/schema.js");
|
||
const { runMigrations } = await import("../src/db/migrate.js");
|
||
const { serversRouter } = await import("../src/routes/servers.js");
|
||
const { syncServerTasks } = await import("../src/services/taskSync.js");
|
||
const { samplesFromSystem, recordServerMetrics, bucketPoints, trendOf, getServerMetrics, MIN_GAP_MS, RETENTION_DAYS, MAX_RANGE_DAYS } = await import("../src/services/serverMetrics.js");
|
||
|
||
await runMigrations();
|
||
|
||
const DAY = 24 * 3600_000;
|
||
const system = (load: number, memUsed: number, diskUsed: number) => ({
|
||
cpu: { load_percent: load },
|
||
memory: { total_bytes: 1000, used_bytes: memUsed },
|
||
disks: [{ mount: "/", size_bytes: 1000, used_bytes: diskUsed }],
|
||
});
|
||
|
||
describe("samplesFromSystem", () => {
|
||
test("turns a report into percentages", () => {
|
||
assert.deepEqual(samplesFromSystem(system(12.34, 250, 900)), [
|
||
{ metric: "cpu", value: 12.3 },
|
||
{ metric: "mem", value: 25 },
|
||
{ metric: "disk:/", value: 90 },
|
||
]);
|
||
});
|
||
|
||
test("leaves out what is missing or makes no sense, and clamps to 0–100", () => {
|
||
assert.deepEqual(samplesFromSystem(null), []);
|
||
assert.deepEqual(samplesFromSystem({ cpu: { load_percent: null }, memory: { total_bytes: 0, used_bytes: 5 }, disks: [{ mount: "/x", size_bytes: 0, used_bytes: 1 }] }), []);
|
||
assert.equal(samplesFromSystem({ cpu: { load_percent: 250 } })[0].value, 100);
|
||
});
|
||
});
|
||
|
||
describe("bucketPoints", () => {
|
||
test("averages points into slices and keeps them in time order", () => {
|
||
const pts = [
|
||
{ t: 0, v: 10 },
|
||
{ t: 10, v: 20 },
|
||
{ t: 90, v: 50 },
|
||
{ t: 100, v: 70 },
|
||
];
|
||
const out = bucketPoints(pts, 0, 100, 2);
|
||
assert.equal(out.length, 2);
|
||
assert.deepEqual(out.map((p) => p.v), [15, 60]);
|
||
assert.ok(out[0].t < out[1].t);
|
||
});
|
||
|
||
test("points outside the range are ignored, and nothing gives nothing", () => {
|
||
assert.deepEqual(bucketPoints([], 0, 100, 4), []);
|
||
assert.deepEqual(bucketPoints([{ t: 500, v: 1 }], 0, 100, 4), []);
|
||
});
|
||
});
|
||
|
||
describe("trendOf", () => {
|
||
const line = (days: number, start: number, perDay: number, n = 20) =>
|
||
Array.from({ length: n }, (_, i) => ({ t: (i / (n - 1)) * days * DAY, v: start + perDay * (i / (n - 1)) * days }));
|
||
|
||
test("a rising disk gets a rate and a forecast", () => {
|
||
const t = trendOf(line(10, 50, 2))!;
|
||
assert.equal(t.perDay, 2);
|
||
assert.equal(t.daysToFull, Math.ceil((100 - 70) / 2));
|
||
});
|
||
|
||
test("a flat or falling one gets no forecast", () => {
|
||
assert.equal(trendOf(line(10, 50, 0))!.daysToFull, null);
|
||
assert.equal(trendOf(line(10, 80, -1))!.daysToFull, null);
|
||
});
|
||
|
||
test("too little data says nothing rather than guessing", () => {
|
||
assert.equal(trendOf(line(10, 50, 2, 4)), null, "fewer than six points");
|
||
assert.equal(trendOf(line(0.5, 50, 2)), null, "less than a day");
|
||
});
|
||
});
|
||
|
||
describe("recording and reading", () => {
|
||
let serverId = 0;
|
||
before(async () => {
|
||
serverId = (await db.insert(servers).values({ name: "elsa", apiTokenHash: "x", apiTokenPrefix: "x" }).returning())[0].id;
|
||
});
|
||
|
||
test("a report stores one sample per figure", async () => {
|
||
const t0 = new Date("2026-10-01T00:00:00Z");
|
||
assert.equal(await recordServerMetrics(serverId, system(10, 100, 500), t0), true);
|
||
const rows = await db.select().from(serverMetrics).where(eq(serverMetrics.serverId, serverId));
|
||
assert.deepEqual(rows.map((r) => r.metric).sort(), ["cpu", "disk:/", "mem"]);
|
||
});
|
||
|
||
test("a report a moment later adds nothing; one after the gap does", async () => {
|
||
const t0 = new Date("2026-10-01T00:00:00Z");
|
||
assert.equal(await recordServerMetrics(serverId, system(11, 100, 500), new Date(t0.getTime() + MIN_GAP_MS - 1000)), false);
|
||
assert.equal(await recordServerMetrics(serverId, system(12, 100, 500), new Date(t0.getTime() + MIN_GAP_MS + 1000)), true);
|
||
});
|
||
|
||
test("samples older than the retention period are dropped when a new one is stored", async () => {
|
||
await db.delete(serverMetrics);
|
||
const old = new Date("2026-01-01T00:00:00Z");
|
||
await recordServerMetrics(serverId, system(1, 1, 1), old);
|
||
const now = new Date(old.getTime() + (RETENTION_DAYS + 1) * DAY);
|
||
await recordServerMetrics(serverId, system(2, 2, 2), now);
|
||
const rows = await db.select().from(serverMetrics).where(eq(serverMetrics.serverId, serverId));
|
||
assert.ok(rows.every((r) => Date.parse(r.recordedAt) >= old.getTime() + DAY), "the old ones are gone");
|
||
assert.equal(rows.length, 3);
|
||
});
|
||
|
||
test("getServerMetrics draws a series per figure, with a forecast for a filling disk", async () => {
|
||
await db.delete(serverMetrics);
|
||
const now = new Date("2026-10-20T00:00:00Z");
|
||
// A disk gaining 2 points a day for ten days, sampled every 6 hours.
|
||
for (let i = 0; i <= 40; i++) {
|
||
const at = new Date(now.getTime() - (10 * DAY - i * 0.25 * DAY));
|
||
await recordServerMetrics(serverId, system(20, 300, 500 + (i / 4) * 20), at);
|
||
}
|
||
const series = await getServerMetrics(serverId, 14, now);
|
||
assert.deepEqual(series.map((s) => s.kind), ["cpu", "mem", "disk"]);
|
||
const disk = series.find((s) => s.kind === "disk")!;
|
||
assert.equal(disk.label, "Disk /");
|
||
assert.equal(disk.latest, 70);
|
||
assert.equal(disk.trend?.perDay, 2);
|
||
assert.equal(disk.trend?.daysToFull, 15);
|
||
assert.equal(series.find((s) => s.kind === "cpu")!.trend, null, "load is not forecast");
|
||
assert.ok(disk.points.length > 5 && disk.points.length <= 96);
|
||
});
|
||
|
||
test("an agent report goes through to the history", async () => {
|
||
await db.delete(serverMetrics);
|
||
await syncServerTasks(serverId, { tasks: [], system: system(33, 400, 600) });
|
||
const rows = await db.select().from(serverMetrics).where(eq(serverMetrics.serverId, serverId));
|
||
assert.equal(rows.length, 3);
|
||
});
|
||
|
||
test("deleting the server takes its history with it", async () => {
|
||
const [other] = await db.insert(servers).values({ name: "gone", apiTokenHash: "y", apiTokenPrefix: "y" }).returning();
|
||
await recordServerMetrics(other.id, system(1, 1, 1), new Date());
|
||
await db.delete(servers).where(eq(servers.id, other.id));
|
||
assert.equal((await db.select().from(serverMetrics).where(eq(serverMetrics.serverId, other.id))).length, 0);
|
||
});
|
||
});
|
||
|
||
describe("GET /api/servers/:id/metrics", () => {
|
||
let app: Server;
|
||
let base = "";
|
||
let serverId = 0;
|
||
|
||
before(async () => {
|
||
await db.insert(users).values({ oidcSub: "tester", name: "Tester", role: "viewer" });
|
||
serverId = (await db.insert(servers).values({ name: "route-test", apiTokenHash: "z", apiTokenPrefix: "z" }).returning())[0].id;
|
||
await recordServerMetrics(serverId, system(5, 100, 100), new Date());
|
||
const a = express();
|
||
a.use(express.json());
|
||
a.use((req, _res, next) => {
|
||
(req as unknown as { session: unknown }).session = { user: { sub: "tester" } };
|
||
next();
|
||
});
|
||
a.use("/api/servers", serversRouter);
|
||
app = await new Promise<Server>((resolve) => {
|
||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||
});
|
||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/servers`;
|
||
});
|
||
|
||
after(() => {
|
||
app.close();
|
||
try {
|
||
rmSync(dir, { recursive: true, force: true });
|
||
} catch {
|
||
// Windows keeps the database file locked until the process exits.
|
||
}
|
||
});
|
||
|
||
test("anyone signed in can read it, and the number of days is kept within what is stored", async () => {
|
||
const res = await fetch(`${base}/${serverId}/metrics?days=9999`);
|
||
assert.equal(res.status, 200);
|
||
assert.equal((await res.json()).days, MAX_RANGE_DAYS, "a year at most: the rest is kept as hours and days");
|
||
const body = await (await fetch(`${base}/${serverId}/metrics?days=${RETENTION_DAYS}`)).json();
|
||
assert.deepEqual(body.series.map((s: { kind: string }) => s.kind), ["cpu", "mem", "disk"]);
|
||
assert.equal((await fetch(`${base}/${serverId}/metrics`)).status, 200);
|
||
});
|
||
|
||
test("an unknown server is a 404 and a bad id a 400", async () => {
|
||
assert.equal((await fetch(`${base}/99999/metrics`)).status, 404);
|
||
assert.equal((await fetch(`${base}/abc/metrics`)).status, 400);
|
||
});
|
||
});
|