Phone card layout, status symbols, breadcrumbs, integration cards and server trends
Tables stack into labelled cards on a phone. Green, red and amber badges carry a check, cross or triangle as well as their colour. Pages inside a menu group show a breadcrumb, with the thing's name on a server's page. Integrations are cards with an Open button and a live status (working, failing, not checked yet, disabled). Agent-reported servers keep a short history of load, memory and disk use (migration 0019, one sample per ten minutes for 35 days) and show it as a Trends card with a days-until-full forecast for a filling disk. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
617dc1ea2f
commit
9965d94036
22 files changed
+3000
-84
No files matched your search
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE `server_metrics` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`server_id` integer NOT NULL,
|
||||
`metric` text NOT NULL,
|
||||
`value` real NOT NULL,
|
||||
`recorded_at` text NOT NULL,
|
||||
FOREIGN KEY (`server_id`) REFERENCES `servers`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `server_metrics_lookup_idx` ON `server_metrics` (`server_id`,`metric`,`recorded_at`);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -134,6 +134,13 @@
|
||||
"when": 1791757001527,
|
||||
"tag": "0018_even_marvel_zombies",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 19,
|
||||
"version": "6",
|
||||
"when": 1792014514813,
|
||||
"tag": "0019_complex_proudstar",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -93,6 +93,22 @@ export const maintenanceAliases = sqliteTable(
|
||||
(table) => [uniqueIndex("maintenance_aliases_unique_idx").on(table.address, table.targetType, table.targetId)],
|
||||
);
|
||||
|
||||
// A short history of each agent-reported server's load, memory and disk use, in percent — see services/serverMetrics.ts.
|
||||
// `metric` is "cpu", "mem" or "disk:<mount point>".
|
||||
export const serverMetrics = sqliteTable(
|
||||
"server_metrics",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
serverId: integer("server_id")
|
||||
.notNull()
|
||||
.references(() => servers.id, { onDelete: "cascade" }),
|
||||
metric: text("metric").notNull(),
|
||||
value: real("value").notNull(),
|
||||
recordedAt: text("recorded_at").notNull(), // ISO
|
||||
},
|
||||
(table) => [index("server_metrics_lookup_idx").on(table.serverId, table.metric, table.recordedAt)],
|
||||
);
|
||||
|
||||
// ─── Settings (key/value) ───────────────────────────────────────────────────
|
||||
|
||||
export const settings = sqliteTable("settings", {
|
||||
|
||||
@@ -24,6 +24,7 @@ import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
|
||||
import { createPbsAdapter } from "../integrations/pbs/adapter.js";
|
||||
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
|
||||
import { attachMatchedServers, summarizeMonitors, toMatchableServers } from "../services/uptimeKumaMatch.js";
|
||||
import { getSourceStatus } from "../services/integrationHealthMonitor.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { ID_PATTERNS, isValidId } from "../utils/pathSegment.js";
|
||||
|
||||
@@ -48,7 +49,13 @@ integrationsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
createdAt: integrations.createdAt,
|
||||
})
|
||||
.from(integrations);
|
||||
res.json({ integrations: rows });
|
||||
// How each one has been doing lately. Calls are logged under the integration's type, so a second integration of the same type shares its streak.
|
||||
res.json({
|
||||
integrations: rows.map((r) => ({
|
||||
...r,
|
||||
health: r.enabled ? getSourceStatus(r.type) : { status: "disabled" as const, consecutiveFailures: 0 },
|
||||
})),
|
||||
});
|
||||
}));
|
||||
|
||||
const configValueSchema = z.union([z.string(), z.boolean()]);
|
||||
|
||||
@@ -6,6 +6,7 @@ import { servers, serverLinks, integrations, dnsRecordsCache, dnsZonesCache, dns
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { generateApiToken } from "../services/tokens.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { ID_PATTERNS } from "../utils/pathSegment.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
@@ -189,6 +190,17 @@ serversRouter.get("/links", asyncHandler(async (_req, res) => {
|
||||
res.json({ links: rows });
|
||||
}));
|
||||
|
||||
// The recent history of an agent-reported server's load, memory and disks, for the trend lines on its page.
|
||||
serversRouter.get("/:id/metrics", asyncHandler(async (req, res) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [server] = await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1);
|
||||
if (!server) return res.status(404).json({ error: "not_found" });
|
||||
const requested = Number(req.query.days);
|
||||
const days = Number.isFinite(requested) ? Math.min(RETENTION_DAYS, Math.max(1, Math.round(requested))) : 7;
|
||||
res.json({ days, series: await getServerMetrics(id, days) });
|
||||
}));
|
||||
|
||||
serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
|
||||
@@ -24,6 +24,18 @@ export function getFailingSources(): { source: string; consecutiveFailures: numb
|
||||
.map(([source, state]) => ({ source, consecutiveFailures: state.consecutiveFailures, alerted: state.alerted }));
|
||||
}
|
||||
|
||||
/**
|
||||
* How a service has been doing since the app started: "failing" while its latest calls keep failing, "ok" once a call has gone
|
||||
* through, and "unknown" when nothing has called it yet (this is a live streak, kept in memory, not a record).
|
||||
*/
|
||||
export function getSourceStatus(source: string): { status: "ok" | "failing" | "unknown"; consecutiveFailures: number } {
|
||||
const state = health.get(source);
|
||||
if (!state) return { status: "unknown", consecutiveFailures: 0 };
|
||||
return state.consecutiveFailures > 0
|
||||
? { status: "failing", consecutiveFailures: state.consecutiveFailures }
|
||||
: { status: "ok", consecutiveFailures: 0 };
|
||||
}
|
||||
|
||||
/** Called after every diagnostic-log entry is recorded, to track consecutive failures per source and alert on threshold-cross / recovery. */
|
||||
export async function trackIntegrationHealth(source: string, ok: boolean): Promise<void> {
|
||||
const state = health.get(source) ?? { consecutiveFailures: 0, alerted: false };
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* A short history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend
|
||||
* (and say how soon a filling disk will run out) instead of only the latest figure.
|
||||
*
|
||||
* One sample is kept per server every ten minutes at most, however often its agent reports, for 35 days. Everything here is a
|
||||
* percentage: CPU load (as the agent reports it), memory in use, and each disk's used share.
|
||||
*/
|
||||
import { and, desc, eq, gte, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { serverMetrics } from "../db/schema.js";
|
||||
import type { IncomingSystemInfo } from "./taskSync.js";
|
||||
|
||||
/** Reports closer together than this add no sample. */
|
||||
export const MIN_GAP_MS = 10 * 60_000;
|
||||
export const RETENTION_DAYS = 35;
|
||||
const DAY_MS = 24 * 3600_000;
|
||||
|
||||
export interface Sample {
|
||||
metric: string;
|
||||
value: number;
|
||||
}
|
||||
|
||||
const round1 = (n: number) => Math.round(n * 10) / 10;
|
||||
const clampPercent = (n: number) => Math.min(100, Math.max(0, n));
|
||||
|
||||
/** The percentages in one agent report. Anything missing or nonsensical (no size, not a number) is left out. */
|
||||
export function samplesFromSystem(system: IncomingSystemInfo | null | undefined): Sample[] {
|
||||
if (!system) return [];
|
||||
const out: Sample[] = [];
|
||||
const load = system.cpu?.load_percent;
|
||||
if (typeof load === "number" && Number.isFinite(load)) out.push({ metric: "cpu", value: round1(clampPercent(load)) });
|
||||
const total = system.memory?.total_bytes;
|
||||
const used = system.memory?.used_bytes;
|
||||
if (typeof total === "number" && typeof used === "number" && total > 0) out.push({ metric: "mem", value: round1(clampPercent((used / total) * 100)) });
|
||||
for (const d of system.disks ?? []) {
|
||||
if (typeof d.mount !== "string" || !d.mount || !(d.size_bytes > 0) || typeof d.used_bytes !== "number") continue;
|
||||
out.push({ metric: `disk:${d.mount}`, value: round1(clampPercent((d.used_bytes / d.size_bytes) * 100)) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Records a report's figures, unless this server was sampled a moment ago. Returns whether anything was stored. */
|
||||
export async function recordServerMetrics(serverId: number, system: IncomingSystemInfo | null | undefined, now: Date = new Date()): Promise<boolean> {
|
||||
const samples = samplesFromSystem(system);
|
||||
if (samples.length === 0) return false;
|
||||
|
||||
const [latest] = await db
|
||||
.select({ at: serverMetrics.recordedAt })
|
||||
.from(serverMetrics)
|
||||
.where(eq(serverMetrics.serverId, serverId))
|
||||
.orderBy(desc(serverMetrics.recordedAt))
|
||||
.limit(1);
|
||||
if (latest && now.getTime() - Date.parse(latest.at) < MIN_GAP_MS) return false;
|
||||
|
||||
const at = now.toISOString();
|
||||
await db.insert(serverMetrics).values(samples.map((s) => ({ serverId, metric: s.metric, value: s.value, recordedAt: at })));
|
||||
await db.delete(serverMetrics).where(and(eq(serverMetrics.serverId, serverId), lt(serverMetrics.recordedAt, new Date(now.getTime() - RETENTION_DAYS * DAY_MS).toISOString())));
|
||||
return true;
|
||||
}
|
||||
|
||||
export interface Point {
|
||||
/** ISO time. */
|
||||
t: string;
|
||||
v: number;
|
||||
}
|
||||
|
||||
/** Averages points into at most `maxPoints` equal slices of [from, to], so a month of samples draws as a short line. */
|
||||
export function bucketPoints(points: { t: number; v: number }[], from: number, to: number, maxPoints: number): Point[] {
|
||||
if (points.length === 0 || to <= from) return [];
|
||||
const size = Math.max((to - from) / maxPoints, 1);
|
||||
const buckets = new Map<number, { sum: number; n: number; tSum: number }>();
|
||||
for (const p of points) {
|
||||
if (p.t < from || p.t > to) continue;
|
||||
const key = Math.min(maxPoints - 1, Math.floor((p.t - from) / size));
|
||||
const b = buckets.get(key) ?? { sum: 0, n: 0, tSum: 0 };
|
||||
b.sum += p.v;
|
||||
b.n += 1;
|
||||
b.tSum += p.t;
|
||||
buckets.set(key, b);
|
||||
}
|
||||
return [...buckets.entries()]
|
||||
.sort((a, b) => a[0] - b[0])
|
||||
.map(([, b]) => ({ t: new Date(b.tSum / b.n).toISOString(), v: round1(b.sum / b.n) }));
|
||||
}
|
||||
|
||||
export interface Trend {
|
||||
/** Change in the percentage per day, from a straight line through the points. */
|
||||
perDay: number;
|
||||
/** For something that fills up: days until it reaches 100% at that rate. Null when it isn't rising or the rate is negligible. */
|
||||
daysToFull: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A straight-line fit over the points. Needs at least six points covering at least a day; with less there is no honest trend to
|
||||
* give. A slope under 0.05 points a day counts as flat, so a disk that wobbles by a fraction isn't forecast to fill in a century.
|
||||
*/
|
||||
export function trendOf(points: { t: number; v: number }[]): Trend | null {
|
||||
if (points.length < 6) return null;
|
||||
const span = points[points.length - 1].t - points[0].t;
|
||||
if (span < DAY_MS) return null;
|
||||
const t0 = points[0].t;
|
||||
const xs = points.map((p) => (p.t - t0) / DAY_MS);
|
||||
const ys = points.map((p) => p.v);
|
||||
const n = points.length;
|
||||
const meanX = xs.reduce((a, b) => a + b, 0) / n;
|
||||
const meanY = ys.reduce((a, b) => a + b, 0) / n;
|
||||
let num = 0;
|
||||
let den = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
num += (xs[i] - meanX) * (ys[i] - meanY);
|
||||
den += (xs[i] - meanX) ** 2;
|
||||
}
|
||||
if (den === 0) return null;
|
||||
const slope = num / den;
|
||||
const perDay = Math.round(slope * 100) / 100;
|
||||
const last = ys[n - 1];
|
||||
const daysToFull = perDay >= 0.05 && last < 100 ? Math.ceil((100 - last) / perDay) : null;
|
||||
return { perDay, daysToFull };
|
||||
}
|
||||
|
||||
export interface MetricSeries {
|
||||
metric: string;
|
||||
kind: "cpu" | "mem" | "disk";
|
||||
label: string;
|
||||
points: Point[];
|
||||
latest: number | null;
|
||||
trend: Trend | null;
|
||||
}
|
||||
|
||||
function describe(metric: string): { kind: MetricSeries["kind"]; label: string } {
|
||||
if (metric === "cpu") return { kind: "cpu", label: "CPU load" };
|
||||
if (metric === "mem") return { kind: "mem", label: "Memory" };
|
||||
return { kind: "disk", label: `Disk ${metric.slice("disk:".length)}` };
|
||||
}
|
||||
|
||||
const ORDER = { cpu: 0, mem: 1, disk: 2 } as const;
|
||||
|
||||
/** Every series recorded for a server over the last `days`, ready to draw. */
|
||||
export async function getServerMetrics(serverId: number, days: number, now: Date = new Date()): Promise<MetricSeries[]> {
|
||||
const to = now.getTime();
|
||||
const from = to - days * DAY_MS;
|
||||
const rows = await db
|
||||
.select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt })
|
||||
.from(serverMetrics)
|
||||
.where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString())))
|
||||
.orderBy(serverMetrics.recordedAt);
|
||||
|
||||
const byMetric = new Map<string, { t: number; v: number }[]>();
|
||||
for (const r of rows) {
|
||||
const list = byMetric.get(r.metric) ?? [];
|
||||
list.push({ t: Date.parse(r.at), v: r.value });
|
||||
byMetric.set(r.metric, list);
|
||||
}
|
||||
|
||||
return [...byMetric.entries()]
|
||||
.map(([metric, raw]) => {
|
||||
const { kind, label } = describe(metric);
|
||||
return {
|
||||
metric,
|
||||
kind,
|
||||
label,
|
||||
points: bucketPoints(raw, from, to, 96),
|
||||
latest: raw[raw.length - 1]?.v ?? null,
|
||||
// A disk filling is the thing worth forecasting; load and memory go up and down by the hour.
|
||||
trend: kind === "disk" ? trendOf(raw) : null,
|
||||
};
|
||||
})
|
||||
.sort((a, b) => ORDER[a.kind] - ORDER[b.kind] || a.label.localeCompare(b.label));
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { and, eq, notInArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { scheduledTasks, servers } from "../db/schema.js";
|
||||
import { recordServerMetrics } from "./serverMetrics.js";
|
||||
|
||||
export interface IncomingTask {
|
||||
scheduleType: "cron" | "systemd_timer" | "windows_task";
|
||||
@@ -121,4 +122,11 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
...(system?.listening_ports ? { listeningPorts: JSON.stringify(system.listening_ports) } : {}),
|
||||
})
|
||||
.where(eq(servers.id, serverId));
|
||||
|
||||
// Keep a little history of the figures too. A failure here must not lose the report itself, which is already saved.
|
||||
try {
|
||||
await recordServerMetrics(serverId, system, new Date(now));
|
||||
} catch (err) {
|
||||
console.error("[serverMetrics] couldn't record a sample:", err);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* What the integration cards say about each integration: working, failing, not checked yet, or disabled.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-int-status-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, integrations } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { integrationsRouter } = await import("../src/routes/integrations.js");
|
||||
const { trackIntegrationHealth, getSourceStatus } = await import("../src/services/integrationHealthMonitor.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
describe("integration health on the list", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values({ oidcSub: "tester", name: "Tester", role: "admin" });
|
||||
await db.insert(integrations).values([
|
||||
{ type: "proxmox", name: "Felicia", baseUrl: "https://pve.lab:8006", config: "{}" },
|
||||
{ type: "gitea", name: "Gitea", baseUrl: "https://git.lab", config: "{}" },
|
||||
{ type: "synology", name: "NAS", baseUrl: "https://nas.lab", config: "{}" },
|
||||
{ type: "tailscale", name: "Off", baseUrl: "https://api.tailscale.com", config: "{}", enabled: false },
|
||||
]);
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "tester" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/integrations", integrationsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/integrations`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("a service nothing has called yet is unknown, not healthy", () => {
|
||||
assert.deepEqual(getSourceStatus("never-called"), { status: "unknown", consecutiveFailures: 0 });
|
||||
});
|
||||
|
||||
test("failing, working, not checked yet and disabled are told apart", async () => {
|
||||
await trackIntegrationHealth("proxmox", false);
|
||||
await trackIntegrationHealth("proxmox", false);
|
||||
await trackIntegrationHealth("gitea", true);
|
||||
|
||||
const list = (await (await fetch(base)).json()).integrations as { name: string; health: { status: string; consecutiveFailures: number } }[];
|
||||
const by = (name: string) => list.find((i) => i.name === name)!.health;
|
||||
assert.deepEqual(by("Felicia"), { status: "failing", consecutiveFailures: 2 });
|
||||
assert.deepEqual(by("Gitea"), { status: "ok", consecutiveFailures: 0 });
|
||||
assert.equal(by("NAS").status, "unknown");
|
||||
assert.equal(by("Off").status, "disabled");
|
||||
});
|
||||
|
||||
test("a call that goes through ends the failing streak", async () => {
|
||||
await trackIntegrationHealth("proxmox", true);
|
||||
assert.deepEqual(getSourceStatus("proxmox"), { status: "ok", consecutiveFailures: 0 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,202 @@
|
||||
/**
|
||||
* The short history kept of each agent-reported server's load, memory and disks, and the trend drawn from it.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-metrics-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { eq } = await import("drizzle-orm");
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, serverMetrics } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { serversRouter } = await import("../src/routes/servers.js");
|
||||
const { syncServerTasks } = await import("../src/services/taskSync.js");
|
||||
const { samplesFromSystem, recordServerMetrics, bucketPoints, trendOf, getServerMetrics, MIN_GAP_MS, RETENTION_DAYS } = await import("../src/services/serverMetrics.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const DAY = 24 * 3600_000;
|
||||
const system = (load: number, memUsed: number, diskUsed: number) => ({
|
||||
cpu: { load_percent: load },
|
||||
memory: { total_bytes: 1000, used_bytes: memUsed },
|
||||
disks: [{ mount: "/", size_bytes: 1000, used_bytes: diskUsed }],
|
||||
});
|
||||
|
||||
describe("samplesFromSystem", () => {
|
||||
test("turns a report into percentages", () => {
|
||||
assert.deepEqual(samplesFromSystem(system(12.34, 250, 900)), [
|
||||
{ metric: "cpu", value: 12.3 },
|
||||
{ metric: "mem", value: 25 },
|
||||
{ metric: "disk:/", value: 90 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("leaves out what is missing or makes no sense, and clamps to 0–100", () => {
|
||||
assert.deepEqual(samplesFromSystem(null), []);
|
||||
assert.deepEqual(samplesFromSystem({ cpu: { load_percent: null }, memory: { total_bytes: 0, used_bytes: 5 }, disks: [{ mount: "/x", size_bytes: 0, used_bytes: 1 }] }), []);
|
||||
assert.equal(samplesFromSystem({ cpu: { load_percent: 250 } })[0].value, 100);
|
||||
});
|
||||
});
|
||||
|
||||
describe("bucketPoints", () => {
|
||||
test("averages points into slices and keeps them in time order", () => {
|
||||
const pts = [
|
||||
{ t: 0, v: 10 },
|
||||
{ t: 10, v: 20 },
|
||||
{ t: 90, v: 50 },
|
||||
{ t: 100, v: 70 },
|
||||
];
|
||||
const out = bucketPoints(pts, 0, 100, 2);
|
||||
assert.equal(out.length, 2);
|
||||
assert.deepEqual(out.map((p) => p.v), [15, 60]);
|
||||
assert.ok(out[0].t < out[1].t);
|
||||
});
|
||||
|
||||
test("points outside the range are ignored, and nothing gives nothing", () => {
|
||||
assert.deepEqual(bucketPoints([], 0, 100, 4), []);
|
||||
assert.deepEqual(bucketPoints([{ t: 500, v: 1 }], 0, 100, 4), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("trendOf", () => {
|
||||
const line = (days: number, start: number, perDay: number, n = 20) =>
|
||||
Array.from({ length: n }, (_, i) => ({ t: (i / (n - 1)) * days * DAY, v: start + perDay * (i / (n - 1)) * days }));
|
||||
|
||||
test("a rising disk gets a rate and a forecast", () => {
|
||||
const t = trendOf(line(10, 50, 2))!;
|
||||
assert.equal(t.perDay, 2);
|
||||
assert.equal(t.daysToFull, Math.ceil((100 - 70) / 2));
|
||||
});
|
||||
|
||||
test("a flat or falling one gets no forecast", () => {
|
||||
assert.equal(trendOf(line(10, 50, 0))!.daysToFull, null);
|
||||
assert.equal(trendOf(line(10, 80, -1))!.daysToFull, null);
|
||||
});
|
||||
|
||||
test("too little data says nothing rather than guessing", () => {
|
||||
assert.equal(trendOf(line(10, 50, 2, 4)), null, "fewer than six points");
|
||||
assert.equal(trendOf(line(0.5, 50, 2)), null, "less than a day");
|
||||
});
|
||||
});
|
||||
|
||||
describe("recording and reading", () => {
|
||||
let serverId = 0;
|
||||
before(async () => {
|
||||
serverId = (await db.insert(servers).values({ name: "elsa", apiTokenHash: "x", apiTokenPrefix: "x" }).returning())[0].id;
|
||||
});
|
||||
|
||||
test("a report stores one sample per figure", async () => {
|
||||
const t0 = new Date("2026-10-01T00:00:00Z");
|
||||
assert.equal(await recordServerMetrics(serverId, system(10, 100, 500), t0), true);
|
||||
const rows = await db.select().from(serverMetrics).where(eq(serverMetrics.serverId, serverId));
|
||||
assert.deepEqual(rows.map((r) => r.metric).sort(), ["cpu", "disk:/", "mem"]);
|
||||
});
|
||||
|
||||
test("a report a moment later adds nothing; one after the gap does", async () => {
|
||||
const t0 = new Date("2026-10-01T00:00:00Z");
|
||||
assert.equal(await recordServerMetrics(serverId, system(11, 100, 500), new Date(t0.getTime() + MIN_GAP_MS - 1000)), false);
|
||||
assert.equal(await recordServerMetrics(serverId, system(12, 100, 500), new Date(t0.getTime() + MIN_GAP_MS + 1000)), true);
|
||||
});
|
||||
|
||||
test("samples older than the retention period are dropped when a new one is stored", async () => {
|
||||
await db.delete(serverMetrics);
|
||||
const old = new Date("2026-01-01T00:00:00Z");
|
||||
await recordServerMetrics(serverId, system(1, 1, 1), old);
|
||||
const now = new Date(old.getTime() + (RETENTION_DAYS + 1) * DAY);
|
||||
await recordServerMetrics(serverId, system(2, 2, 2), now);
|
||||
const rows = await db.select().from(serverMetrics).where(eq(serverMetrics.serverId, serverId));
|
||||
assert.ok(rows.every((r) => Date.parse(r.recordedAt) >= old.getTime() + DAY), "the old ones are gone");
|
||||
assert.equal(rows.length, 3);
|
||||
});
|
||||
|
||||
test("getServerMetrics draws a series per figure, with a forecast for a filling disk", async () => {
|
||||
await db.delete(serverMetrics);
|
||||
const now = new Date("2026-10-20T00:00:00Z");
|
||||
// A disk gaining 2 points a day for ten days, sampled every 6 hours.
|
||||
for (let i = 0; i <= 40; i++) {
|
||||
const at = new Date(now.getTime() - (10 * DAY - i * 0.25 * DAY));
|
||||
await recordServerMetrics(serverId, system(20, 300, 500 + (i / 4) * 20), at);
|
||||
}
|
||||
const series = await getServerMetrics(serverId, 14, now);
|
||||
assert.deepEqual(series.map((s) => s.kind), ["cpu", "mem", "disk"]);
|
||||
const disk = series.find((s) => s.kind === "disk")!;
|
||||
assert.equal(disk.label, "Disk /");
|
||||
assert.equal(disk.latest, 70);
|
||||
assert.equal(disk.trend?.perDay, 2);
|
||||
assert.equal(disk.trend?.daysToFull, 15);
|
||||
assert.equal(series.find((s) => s.kind === "cpu")!.trend, null, "load is not forecast");
|
||||
assert.ok(disk.points.length > 5 && disk.points.length <= 96);
|
||||
});
|
||||
|
||||
test("an agent report goes through to the history", async () => {
|
||||
await db.delete(serverMetrics);
|
||||
await syncServerTasks(serverId, { tasks: [], system: system(33, 400, 600) });
|
||||
const rows = await db.select().from(serverMetrics).where(eq(serverMetrics.serverId, serverId));
|
||||
assert.equal(rows.length, 3);
|
||||
});
|
||||
|
||||
test("deleting the server takes its history with it", async () => {
|
||||
const [other] = await db.insert(servers).values({ name: "gone", apiTokenHash: "y", apiTokenPrefix: "y" }).returning();
|
||||
await recordServerMetrics(other.id, system(1, 1, 1), new Date());
|
||||
await db.delete(servers).where(eq(servers.id, other.id));
|
||||
assert.equal((await db.select().from(serverMetrics).where(eq(serverMetrics.serverId, other.id))).length, 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("GET /api/servers/:id/metrics", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let serverId = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values({ oidcSub: "tester", name: "Tester", role: "viewer" });
|
||||
serverId = (await db.insert(servers).values({ name: "route-test", apiTokenHash: "z", apiTokenPrefix: "z" }).returning())[0].id;
|
||||
await recordServerMetrics(serverId, system(5, 100, 100), new Date());
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "tester" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/servers", serversRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/servers`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("anyone signed in can read it, and the number of days is kept within what is stored", async () => {
|
||||
const res = await fetch(`${base}/${serverId}/metrics?days=9999`);
|
||||
assert.equal(res.status, 200);
|
||||
const body = await res.json();
|
||||
assert.equal(body.days, RETENTION_DAYS);
|
||||
assert.deepEqual(body.series.map((s: { kind: string }) => s.kind), ["cpu", "mem", "disk"]);
|
||||
assert.equal((await fetch(`${base}/${serverId}/metrics`)).status, 200);
|
||||
});
|
||||
|
||||
test("an unknown server is a 404 and a bad id a 400", async () => {
|
||||
assert.equal((await fetch(`${base}/99999/metrics`)).status, 404);
|
||||
assert.equal((await fetch(`${base}/abc/metrics`)).status, 400);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user