A year of history, one chart, and a read API with personal tokens

History: server load, memory and disks are also kept as hourly (four months) and
daily (over two years) averages with lowest and highest; endpoint answers as
hourly (four months) beside the daily figures. Charts ask for 24 hours to a year
and are drawn from whichever is fine enough. A disk that two weeks call flat is
looked at again over ninety days for the forecast. Built once from what is kept.

One chart: TimeChart replaces the sparkline and the multi-line chart: hover and
arrow-key readout, a lowest/highest band, gaps shown as gaps, the same ranges
everywhere and the choice remembered. Used for server trends, comparing servers,
a check's availability and answer times (Endpoints > History) and a server's
availability.

Read API: /api/v1 (servers, metrics, alerts, incidents, inventory, network,
endpoints and their history, maintenance) with personal read-only tokens made
under the user menu: shown once, hashed, 30/90/365 days or none, ten at most,
removed with their owner. GET only. Documented in API.md.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-20 20:22:58 +02:00
1 parent 5e26e77340
commit 68b41b5586
47 files changed
+12360 -247

No files matched your search

@@ -0,0 +1,24 @@
CREATE TABLE `endpoint_hourly` (
`check_id` integer NOT NULL,
`hour` text NOT NULL,
`total` integer DEFAULT 0 NOT NULL,
`ok` integer DEFAULT 0 NOT NULL,
`ms_sum` integer DEFAULT 0 NOT NULL,
`ms_count` integer DEFAULT 0 NOT NULL,
`ms_max` integer DEFAULT 0 NOT NULL,
PRIMARY KEY(`check_id`, `hour`),
FOREIGN KEY (`check_id`) REFERENCES `endpoint_checks`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE TABLE `server_metrics_rollup` (
`server_id` integer NOT NULL,
`metric` text NOT NULL,
`grain` text NOT NULL,
`at` text NOT NULL,
`sum` real NOT NULL,
`n` integer NOT NULL,
`min` real NOT NULL,
`max` real NOT NULL,
PRIMARY KEY(`server_id`, `metric`, `grain`, `at`),
FOREIGN KEY (`server_id`) REFERENCES `servers`(`id`) ON UPDATE no action ON DELETE cascade
);
+13
View File
@@ -0,0 +1,13 @@
CREATE TABLE `api_tokens` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`user_id` integer NOT NULL,
`label` text NOT NULL,
`token_hash` text NOT NULL,
`token_prefix` text NOT NULL,
`created_at` text DEFAULT (current_timestamp) NOT NULL,
`last_used_at` text,
`expires_at` text,
FOREIGN KEY (`user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `api_tokens_token_hash_unique` ON `api_tokens` (`token_hash`);
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+14
View File
@@ -323,6 +323,20 @@
"when": 1792516305790,
"tag": "0045_military_exodus",
"breakpoints": true
},
{
"idx": 46,
"version": "6",
"when": 1792518508025,
"tag": "0046_tough_living_tribunal",
"breakpoints": true
},
{
"idx": 47,
"version": "6",
"when": 1792519188097,
"tag": "0047_eager_dorian_gray",
"breakpoints": true
}
]
}
+5
View File
@@ -47,6 +47,8 @@ import { alertsRouter } from "./routes/alerts.js";
import { mapRouter } from "./routes/map.js";
import { networkRouter } from "./routes/network.js";
import { myWorkRouter } from "./routes/myWork.js";
import { apiTokensRouter } from "./routes/apiTokens.js";
import { publicApiRouter } from "./routes/publicApi.js";
import { generatorRouter } from "./routes/generator.js";
import { alertTicketsRouter } from "./routes/alertTickets.js";
import { avatarsRouter } from "./routes/avatars.js";
@@ -83,6 +85,8 @@ export function createApp() {
app.use("/api/agent/report", agentReportRouter);
// A job reporting that it ran: a token in the address, no session, and no browser to be checked for where it came from.
app.use("/api/heartbeat", heartbeatPingRouter);
// The read API: a personal token in a header, GET only, no session and nothing for the same-origin check to protect.
app.use("/api/v1", publicApiRouter);
// Everything after this is reached with a session cookie, so a request that came from another origin is refused here.
app.use(sameOriginGuard);
app.use(express.json());
@@ -114,6 +118,7 @@ export function createApp() {
}
app.use("/auth", authRouter);
app.use("/api/me/tokens", apiTokensRouter);
app.use("/api/me", meRouter);
app.use("/api/avatars", avatarsRouter);
app.use("/api/heartbeats", heartbeatsRouter);
+57
View File
@@ -263,6 +263,26 @@ export const serverMetrics = sqliteTable(
(table) => [index("server_metrics_lookup_idx").on(table.serverId, table.metric, table.recordedAt)],
);
// The same figures, kept longer and coarser: one row per server, figure and hour (grain "h", kept a few months) or day (grain "d", kept for years), with what is
// needed to draw them — the sum and count (so the average can be worked out as more samples arrive), and the lowest and highest. `at` is the start of the hour
// or day, as ISO. Written as each sample is recorded; see services/serverMetrics.ts.
export const serverMetricsRollup = sqliteTable(
"server_metrics_rollup",
{
serverId: integer("server_id")
.notNull()
.references(() => servers.id, { onDelete: "cascade" }),
metric: text("metric").notNull(),
grain: text("grain").$type<"h" | "d">().notNull(),
at: text("at").notNull(),
sum: real("sum").notNull(),
n: integer("n").notNull(),
min: real("min").notNull(),
max: real("max").notNull(),
},
(t) => [primaryKey({ columns: [t.serverId, t.metric, t.grain, t.at] })],
);
// ─── Settings (key/value) ───────────────────────────────────────────────────
export const settings = sqliteTable("settings", {
@@ -993,6 +1013,24 @@ export const endpointDaily = sqliteTable(
(t) => [primaryKey({ columns: [t.checkId, t.day] })],
);
// The same, by the hour: how many times a check was asked in that hour, how many answered, and the answer times — kept a few months, between the individual
// results (two weeks) and the daily figures (years), so an answer-time chart can reach back beyond two weeks without a point for every check.
export const endpointHourly = sqliteTable(
"endpoint_hourly",
{
checkId: integer("check_id")
.notNull()
.references(() => endpointChecks.id, { onDelete: "cascade" }),
hour: text("hour").notNull(), // ISO, the start of the hour (UTC)
total: integer("total").notNull().default(0),
ok: integer("ok").notNull().default(0),
msSum: integer("ms_sum").notNull().default(0),
msCount: integer("ms_count").notNull().default(0),
msMax: integer("ms_max").notNull().default(0),
},
(t) => [primaryKey({ columns: [t.checkId, t.hour] })],
);
// A certificate, domain registration or secret whose expiry moved later: it was renewed. Written when the app sees it happen, so it only goes back
// as far as the version that began watching (services/renewals.ts).
export const renewals = sqliteTable(
@@ -1050,6 +1088,25 @@ export const metricsTokens = sqliteTable("metrics_tokens", {
lastUsedAt: text("last_used_at"), // ISO
});
// ─── Personal API tokens: read-only access to /api/v1 for scripts and Home Assistant ─────
// One person's own: made by them, shown to them once, and gone when they are. Only a hash of the secret is kept. A token only ever reads (the API has no other verbs),
// and sees what any signed-in person sees.
export const apiTokens = sqliteTable("api_tokens", {
id: integer("id").primaryKey({ autoIncrement: true }),
userId: integer("user_id")
.notNull()
.references(() => users.id, { onDelete: "cascade" }),
label: text("label").notNull(), // e.g. "Home Assistant"
tokenHash: text("token_hash").notNull().unique(),
tokenPrefix: text("token_prefix").notNull(),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
lastUsedAt: text("last_used_at"), // ISO
expiresAt: text("expires_at"), // ISO; null = until removed
});
// ─── Event webhooks: tell another system when something happens ─────────────
export const webhookEvents = ["alert.opened", "alert.resolved", "maintenance.started", "maintenance.ended"] as const;
+4
View File
@@ -25,6 +25,8 @@ import { initEndpointMonitor } from "./services/endpointMonitor.js";
import { initPatchPlans } from "./services/patchPlans.js";
import { initFlapMonitor } from "./services/flapMonitor.js";
import { initUnifiSync } from "./services/unifiSync.js";
import { backfillMetricRollups } from "./services/serverMetrics.js";
import { backfillEndpointHourly } from "./services/endpointChecks.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
import { initIncidentRecorder } from "./services/incidents.js";
import { getAlerts } from "./services/alerts.js";
@@ -69,6 +71,8 @@ initEndpointMonitor();
initPatchPlans();
initFlapMonitor();
initUnifiSync();
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
createApp().listen(env.port, () => {
console.log(`homelab-manager listening on port ${env.port}`);
+46
View File
@@ -0,0 +1,46 @@
import { Router } from "express";
import { z } from "zod";
import { requireAuth } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { createApiToken, EXPIRY_CHOICES, listApiTokens, MAX_TOKENS_PER_PERSON, publicApiToken, removeApiToken, TokenLimitError } from "../services/apiTokens.js";
import { asyncHandler } from "../utils/asyncHandler.js";
/** A person's own API tokens: anyone signed in makes, sees and removes theirs — never anybody else's. */
export const apiTokensRouter = Router();
apiTokensRouter.use(requireAuth);
const createSchema = z.object({
label: z.string().trim().min(1).max(60),
// Days until it stops working; null (or left out) for until it is removed.
expiresInDays: z.union([z.literal(30), z.literal(90), z.literal(365)]).nullable().optional(),
});
apiTokensRouter.get("/", asyncHandler(async (req, res) => {
const rows = await listApiTokens(req.currentUser!.id);
res.json({ tokens: rows.map((r) => publicApiToken(r)), max: MAX_TOKENS_PER_PERSON, expiryChoices: EXPIRY_CHOICES });
}));
// The secret is in this answer and nowhere else, ever.
apiTokensRouter.post("/", asyncHandler(async (req, res) => {
const parsed = createSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give the token a name, like Home Assistant, and choose 30, 90 or 365 days or no end." });
const actor = req.currentUser!;
try {
const { row, token } = await createApiToken({ userId: actor.id, label: parsed.data.label, expiresInDays: parsed.data.expiresInDays ?? null });
await recordAudit({ actor, category: "api_token", action: "create", targetType: "api_token", targetId: row.id, detail: { label: row.label, expiresAt: row.expiresAt } });
res.status(201).json({ token: publicApiToken(row), secret: token });
} catch (err) {
if (err instanceof TokenLimitError) return res.status(400).json({ error: "too_many", message: err.message });
throw err;
}
}));
apiTokensRouter.delete("/:id", asyncHandler(async (req, res) => {
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const actor = req.currentUser!;
const removed = await removeApiToken(actor.id, id);
if (!removed) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor, category: "api_token", action: "delete", targetType: "api_token", targetId: id, detail: { label: removed.label } });
res.status(204).end();
}));
+12 -1
View File
@@ -6,7 +6,7 @@ import { endpointChecks, servers } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { CHECK_KINDS, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
import { CHECK_KINDS, getEndpointHistory, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
import { loadDismissed, loadSuggestions, saveDismissed } from "../services/endpointSuggestions.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -63,6 +63,17 @@ endpointsRouter.get("/", asyncHandler(async (_req, res) => {
});
}));
// How one check has done over a day, a week, a month, three months or a year: availability and answer times, for the chart on its page.
endpointsRouter.get("/:id/history", asyncHandler(async (req, res) => {
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [check] = await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
if (!check) return res.status(404).json({ error: "not_found" });
const requested = Number(req.query.days);
const days = Number.isFinite(requested) ? Math.min(365, Math.max(1, Math.round(requested))) : 7;
res.json({ check, ...(await getEndpointHistory(id, days)) });
}));
function valuesOf(d: z.infer<typeof checkSchema>) {
return {
name: d.name,
+4 -63
View File
@@ -2,12 +2,11 @@ import { Router } from "express";
import { eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { integrations, networkDevices, networkSegments, networkServerLinks, servers } from "../db/schema.js";
import { integrations } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { dependentsOf, loadGraph, needsOfNode, nodeKey } from "../services/dependencies.js";
import { activeSubjects } from "../services/maintenance.js";
import { syncAllUnifi, syncStatuses, syncUnifiIntegration } from "../services/unifiSync.js";
import { buildNetworkOverview } from "../services/networkOverview.js";
import { syncAllUnifi, syncUnifiIntegration } from "../services/unifiSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const networkRouter = Router();
@@ -19,65 +18,7 @@ networkRouter.use(requireAuth);
* names. Everyone signed in can see it (it is the same picture the dependency map and the alerts draw on); only starting a fresh read needs more.
*/
networkRouter.get("/", asyncHandler(async (_req, res) => {
const [intRows, devices, segments, statuses, graph, silenced, linkRows, serverRows] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "unifi")),
db.select().from(networkDevices),
db.select().from(networkSegments),
syncStatuses(),
loadGraph(),
activeSubjects(),
db.select().from(networkServerLinks),
db.select({ id: servers.id, name: servers.name }).from(servers),
]);
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
const intName = new Map(intRows.map((i) => [i.id, i.name]));
const byExternal = new Map(devices.map((d) => [`${d.integrationId}:${d.externalId}`, d]));
// The servers plugged into each device, by what the controller says is connected (see unifiSync.matchClientsToServers).
const plugged = new Map<number, { serverId: number; serverName: string; port: number | null; wired: boolean }[]>();
const serverLinks: { serverId: number; serverName: string; deviceId: number; deviceName: string; port: number | null; wired: boolean }[] = [];
for (const l of linkRows) {
const device = byExternal.get(`${l.integrationId}:${l.deviceExternalId}`);
const name = serverName.get(l.serverId);
if (!device || !name) continue;
plugged.set(device.id, [...(plugged.get(device.id) ?? []), { serverId: l.serverId, serverName: name, port: l.port, wired: l.wired }]);
serverLinks.push({ serverId: l.serverId, serverName: name, deviceId: device.id, deviceName: device.name, port: l.port, wired: l.wired });
}
res.json({
serverLinks,
integrations: intRows.map((i) => ({ id: i.id, name: i.name, enabled: i.enabled, status: statuses[i.id] ?? null })),
devices: devices
.map((d) => {
const key = nodeKey("device", d.id);
const uplink = d.uplinkExternalId ? byExternal.get(`${d.integrationId}:${d.uplinkExternalId}`) : undefined;
return {
id: d.id,
integrationId: d.integrationId,
integrationName: intName.get(d.integrationId) ?? "UniFi",
siteName: d.siteName,
name: d.name,
model: d.model,
kind: d.kind,
ip: d.ip,
mac: d.mac,
state: d.state,
uptimeSeconds: d.uptimeSeconds,
firmware: d.firmware,
firmwareUpdatable: d.firmwareUpdatable,
uplink: uplink ? { id: uplink.id, name: uplink.name } : null,
offlineSince: d.offlineSince,
lastOnlineAt: d.lastOnlineAt,
updatedAt: d.updatedAt,
// How many things depend on it (directly or through others), and how many it needs: what the dependency map says about it.
dependents: dependentsOf(graph.edges, key).length,
needs: needsOfNode(graph.edges, key).length,
inMaintenance: silenced.has(key),
servers: (plugged.get(d.id) ?? []).sort((a, b) => a.serverName.localeCompare(b.serverName)),
};
})
.sort((a, b) => a.name.localeCompare(b.name)),
networks: segments.filter((s) => s.kind === "network").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => (a.vlanId ?? 0) - (b.vlanId ?? 0) || a.name.localeCompare(b.name)),
ssids: segments.filter((s) => s.kind === "ssid").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => a.name.localeCompare(b.name)),
});
res.json(await buildNetworkOverview());
}));
const syncSchema = z.object({ integrationId: z.number().int().positive().optional() });
+165
View File
@@ -0,0 +1,165 @@
import { Router, type NextFunction, type Request, type Response } from "express";
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, servers } from "../db/schema.js";
import { resolveApiToken, type ApiTokenRow, type ApiTokenUser } from "../services/apiTokens.js";
import { getEndpointHistory } from "../services/endpointChecks.js";
import { getIncident, listIncidents } from "../services/incidents.js";
import { buildNetworkOverview } from "../services/networkOverview.js";
import { alertsV1, endpointsV1, inventoryV1, maintenanceV1, oneServerV1, serversV1 } from "../services/publicApi.js";
import { getServerMetrics, MAX_RANGE_DAYS } from "../services/serverMetrics.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { rateLimit } from "../utils/rateLimit.js";
/**
* The read API: JSON about servers, alerts, incidents, inventory, network equipment, endpoint checks and maintenance, for scripts and things like Home Assistant.
* A personal token in the Authorization header (never the address, which ends up in logs); GET only; and nothing here writes anything.
*/
export const publicApiRouter = Router();
declare module "express-serve-static-core" {
interface Request {
apiToken?: { token: ApiTokenRow; user: ApiTokenUser };
}
}
const limiter = rateLimit({ windowMs: 60_000, max: 300, message: "Too many requests." });
publicApiRouter.use(limiter);
publicApiRouter.use((req: Request, res: Response, next: NextFunction) => {
res.setHeader("Cache-Control", "no-store");
if (req.method !== "GET" && req.method !== "HEAD") {
res.setHeader("Allow", "GET, HEAD");
return res.status(405).json({ error: "read_only", message: "This API only reads. It has no way to change anything." });
}
next();
});
publicApiRouter.use(
asyncHandler(async (req, res, next) => {
const match = /^Bearer\s+(\S+)$/i.exec(req.get("authorization") ?? "");
const found = match ? await resolveApiToken(match[1]) : null;
if (!found) {
// The same answer for no token, a wrong one, an expired one and a removed one.
res.setHeader("WWW-Authenticate", 'Bearer realm="homelab-manager"');
return res.status(401).json({ error: "unauthorized", message: "A token is needed: send it as 'Authorization: Bearer <token>'. Make one from the user menu → API tokens." });
}
req.apiToken = found;
next();
}),
);
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
const daysOf = (v: unknown, fallback: number) => {
const n = Number(v);
return Number.isFinite(n) ? Math.min(MAX_RANGE_DAYS, Math.max(1, Math.round(n))) : fallback;
};
publicApiRouter.get("/", (_req, res) => {
res.json({
name: "Homelab Manager read API",
version: 1,
endpoints: [
"GET /api/v1/me",
"GET /api/v1/servers?tag=&status=",
"GET /api/v1/servers/:id",
"GET /api/v1/servers/:id/metrics?days=",
"GET /api/v1/alerts?severity=&category=&acknowledged=&silenced=",
"GET /api/v1/incidents?status=&limit=",
"GET /api/v1/incidents/:id",
"GET /api/v1/inventory",
"GET /api/v1/network",
"GET /api/v1/endpoints",
"GET /api/v1/endpoints/:id/history?days=",
"GET /api/v1/maintenance",
],
});
});
publicApiRouter.get("/me", (req, res) => {
const { token, user } = req.apiToken!;
res.json({ user: { id: user.id, name: user.name, email: user.email, role: user.role }, token: { id: token.id, label: token.label, expiresAt: token.expiresAt } });
});
publicApiRouter.get("/servers", asyncHandler(async (req, res) => {
let list = await serversV1();
if (typeof req.query.tag === "string" && req.query.tag) {
const tag = req.query.tag.toLowerCase();
list = list.filter((s) => s.tags.includes(tag));
}
if (req.query.status === "online" || req.query.status === "offline" || req.query.status === "unknown") list = list.filter((s) => s.status === req.query.status);
res.json({ servers: list });
}));
publicApiRouter.get("/servers/:id", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const server = await oneServerV1(id);
if (!server) return res.status(404).json({ error: "not_found" });
res.json({ server });
}));
publicApiRouter.get("/servers/:id/metrics", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [row] = await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1);
if (!row) return res.status(404).json({ error: "not_found" });
const days = daysOf(req.query.days, 7);
res.json({ days, series: await getServerMetrics(id, days) });
}));
publicApiRouter.get("/alerts", asyncHandler(async (req, res) => {
const report = await alertsV1();
let alerts = report.alerts;
const q = req.query;
if (typeof q.severity === "string") alerts = alerts.filter((a) => a.severity === q.severity);
if (typeof q.category === "string") alerts = alerts.filter((a) => a.category === q.category);
if (q.acknowledged === "true" || q.acknowledged === "false") alerts = alerts.filter((a) => (a.acknowledgement !== null) === (q.acknowledged === "true"));
if (q.silenced === "true" || q.silenced === "false") alerts = alerts.filter((a) => a.silenced === (q.silenced === "true"));
res.json({ ...report, alerts });
}));
publicApiRouter.get("/incidents", asyncHandler(async (req, res) => {
const limit = Math.min(500, Math.max(1, Math.round(Number(req.query.limit)) || 100));
let list = await listIncidents({ limit });
if (req.query.status === "open") list = list.filter((i) => i.open);
else if (req.query.status === "resolved") list = list.filter((i) => !i.open);
res.json({ incidents: list });
}));
publicApiRouter.get("/incidents/:id", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const detail = await getIncident(id);
if (!detail) return res.status(404).json({ error: "not_found" });
res.json(detail);
}));
publicApiRouter.get("/inventory", asyncHandler(async (_req, res) => {
res.json({ assets: await inventoryV1() });
}));
publicApiRouter.get("/network", asyncHandler(async (_req, res) => {
const overview = await buildNetworkOverview();
res.json({ devices: overview.devices, networks: overview.networks, ssids: overview.ssids, serverLinks: overview.serverLinks });
}));
publicApiRouter.get("/endpoints", asyncHandler(async (_req, res) => {
res.json({ endpoints: await endpointsV1() });
}));
publicApiRouter.get("/endpoints/:id/history", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [check] = await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
if (!check) return res.status(404).json({ error: "not_found" });
res.json({ check, ...(await getEndpointHistory(id, daysOf(req.query.days, 7))) });
}));
publicApiRouter.get("/maintenance", asyncHandler(async (_req, res) => {
res.json(await maintenanceV1());
}));
publicApiRouter.use((_req, res) => {
res.status(404).json({ error: "not_found", message: "No such address. GET /api/v1 lists them." });
});
+2 -2
View File
@@ -12,7 +12,7 @@ import { markIgnoredServices, parseHostHealth } from "../services/hostHealth.js"
import { describeDiagnostics, parseDiagnostics } from "../services/agentDiagnostics.js";
import { checkThresholds, parseThresholds } from "../services/serverThresholds.js";
import { latestAgentVersions } from "../services/agentVersions.js";
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
import { getServerMetrics, MAX_RANGE_DAYS } from "../services/serverMetrics.js";
import { getServerActivity } from "../services/serverActivity.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { ID_PATTERNS } from "../utils/pathSegment.js";
@@ -272,7 +272,7 @@ serversRouter.get("/:id/metrics", asyncHandler(async (req, res) => {
const [server] = await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1);
if (!server) return res.status(404).json({ error: "not_found" });
const requested = Number(req.query.days);
const days = Number.isFinite(requested) ? Math.min(RETENTION_DAYS, Math.max(1, Math.round(requested))) : 7;
const days = Number.isFinite(requested) ? Math.min(MAX_RANGE_DAYS, Math.max(1, Math.round(requested))) : 7;
res.json({ days, series: await getServerMetrics(id, days) });
}));
+5 -2
View File
@@ -6,6 +6,9 @@ import { getUptime } from "../services/uptimeHistory.js";
export const uptimeRouter = Router();
// Anyone signed in who can see the server and integration lists can see how those have been doing.
uptimeRouter.get("/", requireAuth, asyncHandler(async (_req, res) => {
res.json(await getUptime());
// `?days=` is 30 by default and at most a year (as far back as the audit log it is worked out from goes).
uptimeRouter.get("/", requireAuth, asyncHandler(async (req, res) => {
const requested = Number(req.query.days);
const days = Number.isFinite(requested) ? Math.min(365, Math.max(1, Math.round(requested))) : undefined;
res.json(await getUptime(new Date(), days));
}));
+2 -2
View File
@@ -12,7 +12,7 @@
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, heartbeats, integrations, networkDevices, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
import { recordFromReport } from "./incidents.js";
@@ -499,7 +499,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
const own = parseThresholds(s.thresholds);
const forecastDays = effectiveThresholds(healthChecks, own).diskForecastDays;
if (!(forecastDays > 0)) continue;
const series = await getServerMetrics(s.id, 14);
const series = await getForecastSeries(s.id);
for (const f of disksFillingSoon(series, forecastDays, (mount) => diskLimit(own, mount, healthChecks.diskUsagePercent))) {
push({
category: "disk",
+74
View File
@@ -0,0 +1,74 @@
/**
* Personal API tokens: read-only access to /api/v1 for scripts, Home Assistant and the like. Each belongs to the person who made it — they see and remove their own —
* and goes when they do. Like an agent token or a share link, the secret is shown once, when it is made, and only its SHA-256 is kept.
*/
import { createHash, randomBytes } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { apiTokens, users } from "../db/schema.js";
export type ApiTokenRow = typeof apiTokens.$inferSelect;
export type ApiTokenUser = typeof users.$inferSelect;
const hash = (token: string) => createHash("sha256").update(token).digest("hex");
/** Most tokens one person may have: one for each thing that reads, and a spare. */
export const MAX_TOKENS_PER_PERSON = 10;
/** How long a token may be asked to last, in days (or never). */
export const EXPIRY_CHOICES = [30, 90, 365] as const;
/** Use is not recorded more than once in this time, so a script that reads every few seconds doesn't write the database each time. */
const USED_WRITE_GAP_MS = 60_000;
export class TokenLimitError extends Error {}
export async function createApiToken(params: { userId: number; label: string; expiresInDays: number | null }, now: Date = new Date()): Promise<{ row: ApiTokenRow; token: string }> {
const existing = await db.select({ id: apiTokens.id }).from(apiTokens).where(eq(apiTokens.userId, params.userId));
if (existing.length >= MAX_TOKENS_PER_PERSON) throw new TokenLimitError(`There are already ${MAX_TOKENS_PER_PERSON} tokens. Remove one you don't use first.`);
const token = `hlm_p_${randomBytes(24).toString("base64url")}`;
const expiresAt = params.expiresInDays === null ? null : new Date(now.getTime() + params.expiresInDays * 86_400_000).toISOString();
const [row] = await db.insert(apiTokens).values({ userId: params.userId, label: params.label, tokenHash: hash(token), tokenPrefix: token.slice(0, 12), expiresAt }).returning();
return { row, token };
}
/** The token a secret belongs to and whose it is, if it is one that still works; notes that it was used. */
export async function resolveApiToken(secret: string, now: Date = new Date()): Promise<{ token: ApiTokenRow; user: ApiTokenUser } | null> {
if (typeof secret !== "string" || secret.length < 20 || secret.length > 128) return null;
const [found] = await db
.select({ token: apiTokens, user: users })
.from(apiTokens)
.innerJoin(users, eq(users.id, apiTokens.userId))
.where(eq(apiTokens.tokenHash, hash(secret)))
.limit(1);
if (!found) return null;
if (found.token.expiresAt && Date.parse(found.token.expiresAt) <= now.getTime()) return null;
if (!found.token.lastUsedAt || now.getTime() - Date.parse(found.token.lastUsedAt) >= USED_WRITE_GAP_MS) {
await db.update(apiTokens).set({ lastUsedAt: now.toISOString() }).where(eq(apiTokens.id, found.token.id));
}
return found;
}
/** A token as its owner's list shows it: everything but the hash. */
export function publicApiToken(row: ApiTokenRow, now: Date = new Date()) {
return {
id: row.id,
label: row.label,
tokenPrefix: row.tokenPrefix,
createdAt: row.createdAt,
lastUsedAt: row.lastUsedAt,
expiresAt: row.expiresAt,
expired: row.expiresAt !== null && Date.parse(row.expiresAt) <= now.getTime(),
};
}
export async function listApiTokens(userId: number) {
return db.select().from(apiTokens).where(eq(apiTokens.userId, userId)).orderBy(apiTokens.createdAt);
}
/** Removes one of a person's own tokens. Returns whether there was one. */
export async function removeApiToken(userId: number, id: number): Promise<ApiTokenRow | null> {
const [row] = await db.select().from(apiTokens).where(and(eq(apiTokens.id, id), eq(apiTokens.userId, userId))).limit(1);
if (!row) return null;
await db.delete(apiTokens).where(eq(apiTokens.id, id));
return row;
}
+1
View File
@@ -39,6 +39,7 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
push: "settings",
share_link: "settings",
metrics_token: "settings",
api_token: "settings",
webhook: "settings",
integration: "integrations",
dns: "integrations",
+147 -1
View File
@@ -14,7 +14,8 @@ import tls from "node:tls";
import type { TLSSocket } from "node:tls";
import { and, asc, eq, gte, inArray, lt, sql } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, endpointDaily, endpointResults } from "../db/schema.js";
import { endpointChecks, endpointDaily, endpointHourly, endpointResults } from "../db/schema.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
import { noteRenewal } from "./renewals.js";
import { resolveTarget } from "../utils/networkGuard.js";
@@ -246,6 +247,7 @@ export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c:
.where(eq(endpointChecks.id, c.id));
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
await addToDaily(c.id, iso, result);
await addToHourly(c.id, iso, result);
// A certificate that is now good for much longer than it was has been renewed.
if (result.certExpiresAt) await noteRenewal("certificate", c.name, c.certExpiresAt, result.certExpiresAt, now);
return result;
@@ -272,6 +274,45 @@ async function addToDaily(checkId: number, atIso: string, result: ProbeResult):
}
}
/** How long the hourly figures are kept: between the individual results (two weeks) and the daily figures (years). */
export const KEEP_HOURLY_DAYS = 120;
/** Adds one result to its hour, so an answer-time chart can reach back beyond two weeks with a point an hour rather than one for every check. */
async function addToHourly(checkId: number, atIso: string, result: ProbeResult): Promise<void> {
const timed = result.ok && result.ms !== null;
const ok = result.ok ? 1 : 0;
const ms = timed ? (result.ms as number) : 0;
const hour = `${atIso.slice(0, 13)}:00:00.000Z`;
try {
await db
.insert(endpointHourly)
.values({ checkId, hour, total: 1, ok, msSum: ms, msCount: timed ? 1 : 0, msMax: ms })
.onConflictDoUpdate({
target: [endpointHourly.checkId, endpointHourly.hour],
set: { total: sql`${endpointHourly.total} + 1`, ok: sql`${endpointHourly.ok} + ${ok}`, msSum: sql`${endpointHourly.msSum} + ${ms}`, msCount: sql`${endpointHourly.msCount} + ${timed ? 1 : 0}`, msMax: sql`max(${endpointHourly.msMax}, ${ms})` },
});
} catch (err) {
console.error("[endpoints] couldn't add to the hourly figures:", err);
}
}
const HOURLY_BACKFILL_FLAG = "endpointHourlyBackfilled";
/** The first time after the hourly figures were added, builds them from the results still kept (two weeks). Done once. */
export async function backfillEndpointHourly(): Promise<boolean> {
if (await getInternalFlag(HOURLY_BACKFILL_FLAG)) return false;
await db.run(sql`
INSERT INTO endpoint_hourly (check_id, hour, total, ok, ms_sum, ms_count, ms_max)
SELECT check_id, substr(at, 1, 13) || ':00:00.000Z', count(*), sum(ok), sum(CASE WHEN ok = 1 AND ms IS NOT NULL THEN ms ELSE 0 END),
sum(CASE WHEN ok = 1 AND ms IS NOT NULL THEN 1 ELSE 0 END), coalesce(max(CASE WHEN ok = 1 AND ms IS NOT NULL THEN ms END), 0)
FROM endpoint_results
GROUP BY check_id, substr(at, 1, 13)
ON CONFLICT (check_id, hour) DO UPDATE SET total = excluded.total, ok = excluded.ok, ms_sum = excluded.ms_sum, ms_count = excluded.ms_count, ms_max = excluded.ms_max
`);
await setInternalFlag(HOURLY_BACKFILL_FLAG, new Date().toISOString());
return true;
}
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
@@ -290,6 +331,111 @@ export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRo
export async function pruneResults(now: Date = new Date()): Promise<void> {
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
await db.delete(endpointDaily).where(lt(endpointDaily.day, new Date(now.getTime() - KEEP_DAILY_DAYS * 86_400_000).toISOString().slice(0, 10)));
await db.delete(endpointHourly).where(lt(endpointHourly.hour, new Date(now.getTime() - KEEP_HOURLY_DAYS * 86_400_000).toISOString()));
}
// ─── How a check has done over a long time ─────────────────────────────────
/** A row of figures for some stretch of time (one result, one hour or one day), with the moment it starts at as epoch ms. */
export interface HistoryRow {
t: number;
total: number;
ok: number;
msSum: number;
msCount: number;
msMax: number;
}
export interface HistoryPoint {
/** ISO. */
t: string;
/** Share of the checks in this stretch that answered, 0–100. */
availability: number | null;
/** Average and longest answer time of the good, timed answers; null when there were none. */
avgMs: number | null;
maxMs: number | null;
/** How many checks it stands for. */
checks: number;
}
/** Rows into at most `maxPoints` equal slices of [from, to]: figures are added up per slice, so a slice is as true as the checks in it. */
export function bucketHistory(rows: HistoryRow[], from: number, to: number, maxPoints: number): HistoryPoint[] {
if (rows.length === 0 || to <= from) return [];
const size = Math.max((to - from) / maxPoints, 1);
const buckets = new Map<number, { total: number; ok: number; msSum: number; msCount: number; msMax: number; tSum: number; rows: number }>();
for (const r of rows) {
if (r.t < from || r.t > to || r.total <= 0) continue;
const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size));
const b = buckets.get(key) ?? { total: 0, ok: 0, msSum: 0, msCount: 0, msMax: 0, tSum: 0, rows: 0 };
b.total += r.total;
b.ok += r.ok;
b.msSum += r.msSum;
b.msCount += r.msCount;
b.msMax = Math.max(b.msMax, r.msMax);
b.tSum += r.t;
b.rows += 1;
buckets.set(key, b);
}
return [...buckets.entries()]
.sort((a, b) => a[0] - b[0])
.map(([, b]) => ({
t: new Date(b.tSum / b.rows).toISOString(),
availability: Math.round((b.ok / b.total) * 1000) / 10,
avgMs: b.msCount > 0 ? Math.round(b.msSum / b.msCount) : null,
maxMs: b.msCount > 0 ? b.msMax : null,
checks: b.total,
}));
}
export interface EndpointHistory {
days: number;
/** Which figures it was drawn from: the results themselves (a day or two), the hourly figures, or the daily ones. */
source: "results" | "hourly" | "daily";
points: HistoryPoint[];
/** Over the whole stretch. */
availability: number | null;
avgMs: number | null;
checks: number;
}
/** How a check did over the last `days` (up to a year): availability and answer times, from the finest figures that reach back that far. */
export async function getEndpointHistory(checkId: number, days: number, now: Date = new Date()): Promise<EndpointHistory> {
days = Math.min(365, Math.max(1, Math.round(days)));
const to = now.getTime();
const from = to - days * 86_400_000;
let rows: HistoryRow[];
let source: EndpointHistory["source"];
if (days <= 2) {
source = "results";
const found = await db
.select({ at: endpointResults.at, ok: endpointResults.ok, ms: endpointResults.ms })
.from(endpointResults)
.where(and(eq(endpointResults.checkId, checkId), gte(endpointResults.at, new Date(from).toISOString())));
rows = found.map((r) => {
const timed = r.ok && r.ms !== null;
return { t: Date.parse(r.at), total: 1, ok: r.ok ? 1 : 0, msSum: timed ? (r.ms as number) : 0, msCount: timed ? 1 : 0, msMax: timed ? (r.ms as number) : 0 };
});
} else if (days <= KEEP_HOURLY_DAYS) {
source = "hourly";
const found = await db.select().from(endpointHourly).where(and(eq(endpointHourly.checkId, checkId), gte(endpointHourly.hour, new Date(from).toISOString())));
rows = found.map((r) => ({ t: Date.parse(r.hour), total: r.total, ok: r.ok, msSum: r.msSum, msCount: r.msCount, msMax: r.msMax }));
} else {
source = "daily";
const found = await db.select().from(endpointDaily).where(and(eq(endpointDaily.checkId, checkId), gte(endpointDaily.day, new Date(from).toISOString().slice(0, 10))));
// A day has no record of its slowest answer; its average stands in for it.
rows = found.map((r) => ({ t: Date.parse(`${r.day}T00:00:00.000Z`), total: r.total, ok: r.ok, msSum: r.msSum, msCount: r.msCount, msMax: r.msCount > 0 ? Math.round(r.msSum / r.msCount) : 0 }));
}
const checks = rows.reduce((n, r) => n + r.total, 0);
const okCount = rows.reduce((n, r) => n + r.ok, 0);
const msCount = rows.reduce((n, r) => n + r.msCount, 0);
return {
days,
source,
points: bucketHistory(rows, from, to, days <= 2 ? 96 : days <= KEEP_HOURLY_DAYS ? 200 : 370),
availability: checks === 0 ? null : Math.round((okCount / checks) * 1000) / 10,
avgMs: msCount === 0 ? null : Math.round(rows.reduce((n, r) => n + r.msSum, 0) / msCount),
checks,
};
}
// ─── What the history says ─────────────────────────────────────────────────
+2 -2
View File
@@ -19,7 +19,7 @@ import { getCachedBackupCoverage } from "./backupCoverage.js";
import { lastAttempt as lastSelfBackup, listSnapshots } from "./selfBackup.js";
import { loadAcks, attentionCounts } from "./alertAcks.js";
import { listIncidents } from "./incidents.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
import { computeSecretStatus } from "./secretStatus.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
@@ -132,7 +132,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
try {
for (const s of serverRows) {
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), 3650, 101)) {
for (const f of disksFillingSoon(await getForecastSeries(s.id, now), 3650, 101)) {
out.add("homelab_server_disk_days_to_full", "gauge", "Days until a disk is full at its present rate of filling; only for disks with a steady climb.", f.daysToFull, { server: s.name, mount: f.mount });
}
}
+72
View File
@@ -0,0 +1,72 @@
/**
* The network equipment read from UniFi controllers: devices with their state, address, uptime and what they are connected through, the networks and the Wi-Fi
* names, and which servers are plugged into which device. Shared by the Network devices page and the read API.
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, networkDevices, networkSegments, networkServerLinks, servers } from "../db/schema.js";
import { dependentsOf, loadGraph, needsOfNode, nodeKey } from "./dependencies.js";
import { activeSubjects } from "./maintenance.js";
import { syncStatuses } from "./unifiSync.js";
export async function buildNetworkOverview() {
const [intRows, devices, segments, statuses, graph, silenced, linkRows, serverRows] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "unifi")),
db.select().from(networkDevices),
db.select().from(networkSegments),
syncStatuses(),
loadGraph(),
activeSubjects(),
db.select().from(networkServerLinks),
db.select({ id: servers.id, name: servers.name }).from(servers),
]);
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
const intName = new Map(intRows.map((i) => [i.id, i.name]));
const byExternal = new Map(devices.map((d) => [`${d.integrationId}:${d.externalId}`, d]));
// The servers plugged into each device, by what the controller says is connected (see unifiSync.matchClientsToServers).
const plugged = new Map<number, { serverId: number; serverName: string; port: number | null; wired: boolean }[]>();
const serverLinks: { serverId: number; serverName: string; deviceId: number; deviceName: string; port: number | null; wired: boolean }[] = [];
for (const l of linkRows) {
const device = byExternal.get(`${l.integrationId}:${l.deviceExternalId}`);
const name = serverName.get(l.serverId);
if (!device || !name) continue;
plugged.set(device.id, [...(plugged.get(device.id) ?? []), { serverId: l.serverId, serverName: name, port: l.port, wired: l.wired }]);
serverLinks.push({ serverId: l.serverId, serverName: name, deviceId: device.id, deviceName: device.name, port: l.port, wired: l.wired });
}
return {
serverLinks,
integrations: intRows.map((i) => ({ id: i.id, name: i.name, enabled: i.enabled, status: statuses[i.id] ?? null })),
devices: devices
.map((d) => {
const key = nodeKey("device", d.id);
const uplink = d.uplinkExternalId ? byExternal.get(`${d.integrationId}:${d.uplinkExternalId}`) : undefined;
return {
id: d.id,
integrationId: d.integrationId,
integrationName: intName.get(d.integrationId) ?? "UniFi",
siteName: d.siteName,
name: d.name,
model: d.model,
kind: d.kind,
ip: d.ip,
mac: d.mac,
state: d.state,
uptimeSeconds: d.uptimeSeconds,
firmware: d.firmware,
firmwareUpdatable: d.firmwareUpdatable,
uplink: uplink ? { id: uplink.id, name: uplink.name } : null,
offlineSince: d.offlineSince,
lastOnlineAt: d.lastOnlineAt,
updatedAt: d.updatedAt,
// How many things depend on it (directly or through others), and how many it needs: what the dependency map says about it.
dependents: dependentsOf(graph.edges, key).length,
needs: needsOfNode(graph.edges, key).length,
inMaintenance: silenced.has(key),
servers: (plugged.get(d.id) ?? []).sort((a, b) => a.serverName.localeCompare(b.serverName)),
};
})
.sort((a, b) => a.name.localeCompare(b.name)),
networks: segments.filter((s) => s.kind === "network").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => (a.vlanId ?? 0) - (b.vlanId ?? 0) || a.name.localeCompare(b.name)),
ssids: segments.filter((s) => s.kind === "ssid").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => a.name.localeCompare(b.name)),
};
}
+163
View File
@@ -0,0 +1,163 @@
/**
* What the read API (/api/v1) says. Each of these is a deliberate, stable shape — not the app's own page payloads, which change with the pages — and only things
* any signed-in person can already see: no tokens, no secrets, no passwords, no settings. Dates are ISO 8601 in UTC; sizes are bytes; percentages are 0–100.
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, endpointChecks, servers } from "../db/schema.js";
import { env } from "../env.js";
import { attentionCounts, loadAcks } from "./alertAcks.js";
import { getAlerts } from "./alerts.js";
import { loadStats } from "./endpointChecks.js";
import { describeTarget, listActiveWindows, listUpcomingWindows } from "./maintenance.js";
import { parseHostHealth } from "./hostHealth.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { parseTags } from "./serverTags.js";
import { parseThresholds } from "./serverThresholds.js";
import { getSettings } from "./settingsStore.js";
import { serverStatus } from "./topology.js";
/** An address inside the app as a full address, for a script that has no idea where the app lives. */
export function absoluteUrl(path: string | null): string | null {
return path === null ? null : `${env.appBaseUrl.replace(/\/+$/, "")}${path.startsWith("/") ? "" : "/"}${path}`;
}
const json = <T>(text: string | null, fallback: T): T => {
if (!text) return fallback;
try {
return JSON.parse(text) as T;
} catch {
return fallback;
}
};
type ServerRow = typeof servers.$inferSelect;
export function serverV1(row: ServerRow, offlineMinutes: number, now: Date) {
const status = serverStatus(row, offlineMinutes, now);
const disks = json<{ mount: string; sizeBytes: number; usedBytes: number }[]>(row.disks, []);
const updates = parsePendingUpdates(row.pendingUpdates);
const health = parseHostHealth(row.hostHealth);
return {
id: row.id,
name: row.name,
hostname: row.hostname,
description: row.description,
os: row.osType,
tags: parseTags(row.tags),
ipAddresses: json<string[]>(row.ipAddresses, []),
// "online" while its agent has reported within its limit, "offline" after, "unknown" if it never has.
status: status === "ok" ? "online" : status === "bad" ? "offline" : "unknown",
lastSeenAt: row.lastSeenAt,
createdAt: row.createdAt,
agentVersion: row.agentVersion ?? null,
runsOnServerId: row.parentServerId ?? null,
hardware: { manufacturer: row.hwManufacturer ?? null, model: row.hwModel ?? null, serial: row.hwSerial ?? null },
cpu: { model: row.cpuModel, cores: row.cpuCores, loadPercent: row.cpuLoadPercent },
memory: { totalBytes: row.memTotalBytes, usedBytes: row.memUsedBytes },
disks: disks.map((d) => ({ mount: d.mount, sizeBytes: d.sizeBytes, usedBytes: d.usedBytes, usedPercent: d.sizeBytes > 0 ? Math.round((d.usedBytes / d.sizeBytes) * 1000) / 10 : null })),
updates: updates ? { manager: updates.manager, count: updates.count, security: updates.security ?? null, rebootRequired: !!updates.rebootRequired, reportedAt: updates.reportedAt ?? null } : null,
health: health ? { failedUnits: health.failedUnits?.length ?? 0, reportedAt: health.reportedAt ?? null } : null,
thresholds: parseThresholds(row.thresholds),
url: absoluteUrl(`/servers/${row.id}`),
};
}
export async function serversV1(now: Date = new Date()) {
const [rows, settings] = await Promise.all([db.select().from(servers).orderBy(servers.name), getSettings()]);
return rows.map((r) => serverV1(r, settings.healthChecks.serverOfflineMinutes, now));
}
export async function oneServerV1(id: number, now: Date = new Date()) {
const [row] = await db.select().from(servers).where(eq(servers.id, id)).limit(1);
if (!row) return null;
return serverV1(row, (await getSettings()).healthChecks.serverOfflineMinutes, now);
}
export async function alertsV1() {
const [report, acks] = await Promise.all([getAlerts(false), loadAcks()]);
const attention = attentionCounts(report.alerts, acks);
return {
generatedAt: report.generatedAt,
// What is on the page: those nobody has looked at, by severity, and how many somebody has.
needingAttention: { critical: attention.critical, warning: attention.warning, acknowledged: attention.acknowledged },
counts: report.counts,
couldntCheck: report.couldntCheck,
alerts: report.alerts.map((a) => {
const ack = acks.get(a.id);
return {
id: a.id,
severity: a.severity,
category: a.category,
source: a.source,
message: a.message,
url: absoluteUrl(a.link),
silenced: a.silenced,
flapping: !!a.flapping,
target: a.target,
causedBy: a.causedBy ?? null,
affects: a.affects ?? [],
acknowledgement: ack?.active ? { by: ack.ackedBy, at: ack.ackedAt, note: ack.note, until: ack.until, assignee: ack.assignee?.name ?? null } : null,
};
}),
};
}
export async function inventoryV1() {
const [rows, serverRows] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select({ id: servers.id, name: servers.name }).from(servers)]);
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
return rows.map((a) => ({
id: a.id,
name: a.name,
kind: a.kind,
manufacturer: a.manufacturer,
model: a.model,
serial: a.serial,
location: a.location,
rackUnit: a.rackUnit,
purchaseDate: a.purchaseDate,
purchasePrice: a.purchasePrice,
warrantyUntil: a.warrantyUntil,
endOfSupport: a.endOfSupport,
replacementCost: a.replacementCost,
powerWatts: a.powerWatts,
notes: a.notes,
server: a.serverId !== null && serverName.has(a.serverId) ? { id: a.serverId, name: serverName.get(a.serverId)! } : null,
updatedAt: a.updatedAt,
}));
}
export async function maintenanceV1(now: Date = new Date()) {
const shape = async (w: Awaited<ReturnType<typeof listActiveWindows>>[number]) => {
const target = await describeTarget(w.targetType, w.targetId);
return target ? { id: w.id, target: { type: w.targetType, id: w.targetId, name: target.name, kind: target.kind }, reason: w.reason, startsAt: w.startedAt, endsAt: w.endsAt, createdBy: w.createdBy } : null;
};
const active = (await Promise.all((await listActiveWindows(now)).map(shape))).filter((w): w is NonNullable<typeof w> => w !== null);
const upcoming = (await Promise.all((await listUpcomingWindows(now)).map(shape))).filter((w): w is NonNullable<typeof w> => w !== null);
return { active, upcoming };
}
export async function endpointsV1(now: Date = new Date()) {
const rows = await db.select().from(endpointChecks).orderBy(endpointChecks.name);
const stats = await loadStats(rows.map((r) => r.id), now);
return rows.map((r) => {
const st = stats.get(r.id);
return {
id: r.id,
name: r.name,
kind: r.kind,
target: r.target,
port: r.port,
enabled: r.enabled,
status: !r.enabled ? "paused" : r.lastOk === null ? "unknown" : r.lastOk ? "up" : "down",
lastCheckedAt: r.lastCheckedAt,
lastResponseMs: r.lastResponseMs,
lastMessage: r.lastOk === false ? r.lastMessage : null,
certificateExpiresAt: r.certExpiresAt,
uptime24hPercent: st?.uptime24h ?? null,
uptime7dPercent: st?.uptime7d ?? null,
averageResponseMs24h: st?.avgMs24h ?? null,
serverId: r.serverId,
};
});
}
+174 -19
View File
@@ -1,19 +1,30 @@
/**
* A short history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend
* A history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend
* (and say how soon a filling disk will run out) instead of only the latest figure.
*
* One sample is kept per server every ten minutes at most, however often its agent reports, for 35 days. Everything here is a
* percentage: CPU load (as the agent reports it), memory in use, and each disk's used share.
*
* Beyond those 35 days the same figures are kept coarser: each sample also goes into its hour (kept four months) and its day (kept over two years), as a
* sum, a count, a lowest and a highest. A chart asks for a number of days and is answered from whichever is fine enough and still there: the samples up to
* 35 days, the hours up to 120, the days beyond. A year of a disk filling is then one point a day.
*/
import { and, desc, eq, gte, lt } from "drizzle-orm";
import { and, desc, eq, gte, lt, sql } from "drizzle-orm";
import { db } from "../db/client.js";
import { serverMetrics } from "../db/schema.js";
import { serverMetrics, serverMetricsRollup } from "../db/schema.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
import type { IncomingSystemInfo } from "./taskSync.js";
/** Reports closer together than this add no sample. */
export const MIN_GAP_MS = 10 * 60_000;
export const RETENTION_DAYS = 35;
/** How long the hourly and daily figures are kept. */
export const HOURLY_KEEP_DAYS = 120;
export const DAILY_KEEP_DAYS = 800;
/** The longest stretch a chart may ask for. */
export const MAX_RANGE_DAYS = 365;
const DAY_MS = 24 * 3600_000;
const HOUR_MS = 3600_000;
export interface Sample {
metric: string;
@@ -39,6 +50,35 @@ export function samplesFromSystem(system: IncomingSystemInfo | null | undefined)
return out;
}
/** The start of the hour or day a moment falls in, as ISO. */
export const hourStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / HOUR_MS) * HOUR_MS).toISOString();
export const dayStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / DAY_MS) * DAY_MS).toISOString();
/** Adds samples to their hour and day. Never allowed to break recording the sample itself. */
async function addToRollups(serverId: number, samples: Sample[], at: Date): Promise<void> {
try {
for (const grain of ["h", "d"] as const) {
const bucket = grain === "h" ? hourStart(at.getTime()) : dayStart(at.getTime());
for (const s of samples) {
await db
.insert(serverMetricsRollup)
.values({ serverId, metric: s.metric, grain, at: bucket, sum: s.value, n: 1, min: s.value, max: s.value })
.onConflictDoUpdate({
target: [serverMetricsRollup.serverId, serverMetricsRollup.metric, serverMetricsRollup.grain, serverMetricsRollup.at],
set: {
sum: sql`${serverMetricsRollup.sum} + ${s.value}`,
n: sql`${serverMetricsRollup.n} + 1`,
min: sql`min(${serverMetricsRollup.min}, ${s.value})`,
max: sql`max(${serverMetricsRollup.max}, ${s.value})`,
},
});
}
}
} catch (err) {
console.error("[metrics] couldn't add to the hourly and daily figures:", err);
}
}
/** Records a report's figures, unless this server was sampled a moment ago. Returns whether anything was stored. */
export async function recordServerMetrics(serverId: number, system: IncomingSystemInfo | null | undefined, now: Date = new Date()): Promise<boolean> {
const samples = samplesFromSystem(system);
@@ -54,7 +94,32 @@ export async function recordServerMetrics(serverId: number, system: IncomingSyst
const at = now.toISOString();
await db.insert(serverMetrics).values(samples.map((s) => ({ serverId, metric: s.metric, value: s.value, recordedAt: at })));
await addToRollups(serverId, samples, now);
await db.delete(serverMetrics).where(and(eq(serverMetrics.serverId, serverId), lt(serverMetrics.recordedAt, new Date(now.getTime() - RETENTION_DAYS * DAY_MS).toISOString())));
await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "h"), lt(serverMetricsRollup.at, new Date(now.getTime() - HOURLY_KEEP_DAYS * DAY_MS).toISOString())));
await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "d"), lt(serverMetricsRollup.at, new Date(now.getTime() - DAILY_KEEP_DAYS * DAY_MS).toISOString())));
return true;
}
const BACKFILL_FLAG = "metricsRollupBackfilled";
/**
* The first time after the hourly and daily figures were added, builds them from the samples already kept (up to 35 days), so the longer ranges are not empty
* for weeks. A bucket is written from what the samples say, replacing what is there, so it is right however far it got before. Done once.
*/
export async function backfillMetricRollups(): Promise<boolean> {
if (await getInternalFlag(BACKFILL_FLAG)) return false;
for (const [grain, len] of [["h", 13], ["d", 10]] as const) {
const suffix = grain === "h" ? ":00:00.000Z" : "T00:00:00.000Z";
await db.run(sql`
INSERT INTO server_metrics_rollup (server_id, metric, grain, at, sum, n, min, max)
SELECT server_id, metric, ${grain}, substr(recorded_at, 1, ${len}) || ${suffix}, sum(value), count(*), min(value), max(value)
FROM server_metrics
GROUP BY server_id, metric, substr(recorded_at, 1, ${len})
ON CONFLICT (server_id, metric, grain, at) DO UPDATE SET sum = excluded.sum, n = excluded.n, min = excluded.min, max = excluded.max
`);
}
await setInternalFlag(BACKFILL_FLAG, new Date().toISOString());
return true;
}
@@ -62,6 +127,9 @@ export interface Point {
/** ISO time. */
t: string;
v: number;
/** The lowest and highest figure in what this point stands for, when it stands for several (hourly and daily figures). */
lo?: number;
hi?: number;
}
/** Averages points into at most `maxPoints` equal slices of [from, to], so a month of samples draws as a short line. */
@@ -83,6 +151,42 @@ export function bucketPoints(points: { t: number; v: number }[], from: number, t
.map(([, b]) => ({ t: new Date(b.tSum / b.n).toISOString(), v: round1(b.sum / b.n) }));
}
/** A row of the hourly or daily figures, with the moment it starts at as epoch ms. */
export interface RollupRow {
t: number;
sum: number;
n: number;
min: number;
max: number;
}
/** The average of what each row stands for. */
export function rollupAverages(rows: RollupRow[]): { t: number; v: number }[] {
return rows.filter((r) => r.n > 0).map((r) => ({ t: r.t, v: r.sum / r.n }));
}
/** Rows into at most `maxPoints` slices of [from, to]: the average of the samples in each (not of the rows), with the lowest and highest. */
export function bucketRollups(rows: RollupRow[], from: number, to: number, maxPoints: number): Point[] {
if (rows.length === 0 || to <= from) return [];
const size = Math.max((to - from) / maxPoints, 1);
const buckets = new Map<number, { sum: number; n: number; lo: number; hi: number; tSum: number; rows: number }>();
for (const r of rows) {
if (r.t < from || r.t > to || r.n <= 0) continue;
const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size));
const b = buckets.get(key) ?? { sum: 0, n: 0, lo: Infinity, hi: -Infinity, tSum: 0, rows: 0 };
b.sum += r.sum;
b.n += r.n;
b.lo = Math.min(b.lo, r.min);
b.hi = Math.max(b.hi, r.max);
b.tSum += r.t;
b.rows += 1;
buckets.set(key, b);
}
return [...buckets.entries()]
.sort((a, b) => a[0] - b[0])
.map(([, b]) => ({ t: new Date(b.tSum / b.rows).toISOString(), v: round1(b.sum / b.n), lo: round1(b.lo), hi: round1(b.hi) }));
}
export interface Trend {
/** Change in the percentage per day, from a straight line through the points. */
perDay: number;
@@ -175,35 +279,86 @@ function describe(metric: string): { kind: MetricSeries["kind"]; label: string }
const ORDER = { cpu: 0, mem: 1, disk: 2 } as const;
/** Every series recorded for a server over the last `days`, ready to draw. */
/** Which of the kept figures answers a stretch of `days`: the samples while they reach back that far, then the hours, then the days. */
export function sourceFor(days: number): "samples" | "hourly" | "daily" {
return days <= RETENTION_DAYS ? "samples" : days <= HOURLY_KEEP_DAYS ? "hourly" : "daily";
}
/** How many points a chart over `days` is drawn from at most. */
const pointsFor = (days: number) => (days <= RETENTION_DAYS ? 96 : days <= HOURLY_KEEP_DAYS ? 200 : MAX_RANGE_DAYS + 5);
/** The latest sample of each figure of a server, whatever range is asked for. */
async function latestSamples(serverId: number): Promise<Map<string, number>> {
const rows = await db.all<{ metric: string; value: number }>(
sql`SELECT metric, value FROM server_metrics WHERE id IN (SELECT max(id) FROM server_metrics WHERE server_id = ${serverId} GROUP BY metric)`,
);
return new Map(rows.map((r) => [r.metric, r.value]));
}
/** Every series recorded for a server over the last `days` (up to a year), ready to draw. */
export async function getServerMetrics(serverId: number, days: number, now: Date = new Date()): Promise<MetricSeries[]> {
days = Math.min(MAX_RANGE_DAYS, Math.max(1, days));
const to = now.getTime();
const from = to - days * DAY_MS;
const rows = await db
.select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt })
.from(serverMetrics)
.where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString())))
.orderBy(serverMetrics.recordedAt);
const source = sourceFor(days);
const byMetric = new Map<string, { t: number; v: number }[]>();
for (const r of rows) {
const list = byMetric.get(r.metric) ?? [];
list.push({ t: Date.parse(r.at), v: r.value });
byMetric.set(r.metric, list);
// The figures, per metric, as { t, v } for a trend and as points for the chart.
const raw = new Map<string, { t: number; v: number }[]>();
const points = new Map<string, Point[]>();
if (source === "samples") {
const rows = await db
.select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt })
.from(serverMetrics)
.where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString())))
.orderBy(serverMetrics.recordedAt);
for (const r of rows) raw.set(r.metric, [...(raw.get(r.metric) ?? []), { t: Date.parse(r.at), v: r.value }]);
for (const [metric, list] of raw) points.set(metric, bucketPoints(list, from, to, pointsFor(days)));
} else {
const rows = await db
.select()
.from(serverMetricsRollup)
.where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, source === "hourly" ? "h" : "d"), gte(serverMetricsRollup.at, new Date(from).toISOString())))
.orderBy(serverMetricsRollup.at);
const byMetric = new Map<string, RollupRow[]>();
for (const r of rows) byMetric.set(r.metric, [...(byMetric.get(r.metric) ?? []), { t: Date.parse(r.at), sum: r.sum, n: r.n, min: r.min, max: r.max }]);
for (const [metric, list] of byMetric) {
raw.set(metric, rollupAverages(list));
points.set(metric, bucketRollups(list, from, to, pointsFor(days)));
}
}
const latest = source === "samples" ? null : await latestSamples(serverId);
return [...byMetric.entries()]
.map(([metric, raw]) => {
return [...raw.entries()]
.map(([metric, list]) => {
const { kind, label } = describe(metric);
const newest = latest?.get(metric) ?? list[list.length - 1]?.v;
return {
metric,
kind,
label,
points: bucketPoints(raw, from, to, 96),
latest: raw[raw.length - 1]?.v ?? null,
points: points.get(metric) ?? [],
latest: newest === undefined ? null : source === "samples" ? newest : round1(newest),
// A disk filling is the thing worth forecasting; load and memory go up and down by the hour.
trend: kind === "disk" ? trendOf(raw) : null,
trend: kind === "disk" ? trendOf(list) : null,
};
})
.sort((a, b) => ORDER[a.kind] - ORDER[b.kind] || a.label.localeCompare(b.label));
}
/** How far back the longer look at a disk goes when the last two weeks say it is flat. */
export const LONG_FORECAST_DAYS = 90;
/**
* The series to forecast from: the last two weeks, as before — except that a disk those two weeks call flat is looked at again over the last
* ninety days, so a slow, steady climb (a point a week) is seen too. The longer trend replaces the short one only if it is really rising.
*/
export async function getForecastSeries(serverId: number, now: Date = new Date()): Promise<MetricSeries[]> {
const short = await getServerMetrics(serverId, 14, now);
if (!short.some((s) => s.kind === "disk" && (!s.trend || s.trend.daysToFull === null))) return short;
const long = await getServerMetrics(serverId, LONG_FORECAST_DAYS, now);
return short.map((s) => {
if (s.kind !== "disk" || (s.trend && s.trend.daysToFull !== null)) return s;
const longer = long.find((l) => l.metric === s.metric);
return longer?.trend && longer.trend.daysToFull !== null ? { ...s, trend: longer.trend } : s;
});
}
+2 -2
View File
@@ -16,7 +16,7 @@ import { describeTarget, listUpcomingWindows } from "./maintenance.js";
import { notify, notifyNow } from "./notify.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { computeSecretStatus } from "./secretStatus.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
@@ -163,7 +163,7 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
const own = parseThresholds(s.thresholds);
const forecastDays = effectiveThresholds(settings.healthChecks, own).diskForecastDays;
if (!(forecastDays > 0)) continue;
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
for (const f of disksFillingSoon(await getForecastSeries(s.id, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
}
}
+249
View File
@@ -0,0 +1,249 @@
/**
* Longer history: server figures and endpoint answer times kept by the hour and the day beyond the days they are kept in full, and drawn from whichever is
* fine enough for the range asked for, up to a year.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-history-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, serverMetrics, serverMetricsRollup, endpointChecks, endpointResults, endpointHourly, endpointDaily } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const metrics = await import("../src/services/serverMetrics.js");
const checks = await import("../src/services/endpointChecks.js");
const { serversRouter } = await import("../src/routes/servers.js");
const { endpointsRouter } = await import("../src/routes/endpoints.js");
const { eq, and } = await import("drizzle-orm");
await runMigrations();
const DAY = 24 * 3600_000;
const HOUR = 3600_000;
const NOW = new Date(Math.floor(Date.now() / HOUR) * HOUR + 20 * 60_000); // twenty minutes into an hour
const ago = (days: number, extraMs = 0) => new Date(NOW.getTime() - days * DAY + extraMs);
const system = (cpu: number, disk: number) => ({ cpu: { load_percent: cpu }, memory: { total_bytes: 1000, used_bytes: 500 }, disks: [{ mount: "/", size_bytes: 100, used_bytes: disk }] }) as never;
let serverId = 0;
let checkId = 0;
before(async () => {
await db.insert(users).values([{ oidcSub: "ada", name: "Ada", role: "admin" }]);
serverId = (await db.insert(servers).values({ name: "web", apiTokenHash: "w", apiTokenPrefix: "w", lastSeenAt: NOW.toISOString() }).returning({ id: servers.id }))[0].id;
checkId = (await db.insert(endpointChecks).values({ name: "Site", kind: "tcp", target: "127.0.0.1", port: 1 }).returning({ id: endpointChecks.id }))[0].id;
});
after(() => {
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows may still hold the database file
}
});
describe("the hourly and daily figures of a server", () => {
test("which figures answer which range: the samples while they reach, then the hours, then the days", () => {
assert.deepEqual([1, 7, 35, 36, 120, 121, 365].map(metrics.sourceFor), ["samples", "samples", "samples", "hourly", "hourly", "daily", "daily"]);
});
test("a sample goes into its hour and its day, and the next one in the same hour is added to it, with the lowest and highest kept", async () => {
const t = new Date(ago(0).getTime() - 10 * 60_000); // ten minutes into the hour
assert.equal(await metrics.recordServerMetrics(serverId, system(10, 50), t), true);
assert.equal(await metrics.recordServerMetrics(serverId, system(30, 52), new Date(t.getTime() + 11 * 60_000)), true);
const cpuHour = (await db.select().from(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.metric, "cpu"), eq(serverMetricsRollup.grain, "h"))))[0];
assert.deepEqual([cpuHour.n, cpuHour.sum, cpuHour.min, cpuHour.max, cpuHour.at], [2, 40, 10, 30, metrics.hourStart(t.getTime())]);
const cpuDay = (await db.select().from(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.metric, "cpu"), eq(serverMetricsRollup.grain, "d"))))[0];
assert.deepEqual([cpuDay.n, cpuDay.at], [2, metrics.dayStart(t.getTime())]);
});
test("a hundred days of a disk filling: seven days come from the samples, ninety from the hours, a year from the days", async () => {
// one sample a day, climbing 0.3 points a day, oldest first (the way agents report)
const hist = (await db.insert(servers).values({ name: "hist", apiTokenHash: "h", apiTokenPrefix: "h", lastSeenAt: NOW.toISOString() }).returning({ id: servers.id }))[0].id;
for (let d = 100; d >= 1; d--) await metrics.recordServerMetrics(hist, system(20, 30 + (100 - d) * 0.3), ago(d));
const week = await metrics.getServerMetrics(hist, 7, NOW);
const disk7 = week.find((s) => s.metric === "disk:/")!;
assert.ok(disk7.points.length > 0 && disk7.points.every((p) => p.lo === undefined), "from the samples: no band");
const ninety = await metrics.getServerMetrics(hist, 90, NOW);
const disk90 = ninety.find((s) => s.metric === "disk:/")!;
assert.ok(disk90.points.length >= 85, `about a point a day (got ${disk90.points.length})`);
assert.ok(disk90.points.every((p) => p.lo !== undefined && p.hi !== undefined && p.lo <= p.v && p.v <= p.hi), "from the hours: each has a lowest and highest");
assert.ok(disk90.trend && Math.abs(disk90.trend.perDay - 0.3) < 0.05, "and the trend reaches back all ninety days");
assert.ok(disk90.trend!.spanDays > 85);
const year = await metrics.getServerMetrics(hist, 365, NOW);
const diskYear = year.find((s) => s.metric === "disk:/")!;
assert.ok(diskYear.points.length >= 99 && diskYear.points.length <= 370);
assert.equal(diskYear.latest, 30 + 99 * 0.3 > 100 ? 100 : Math.round((30 + 99 * 0.3) * 10) / 10, "the latest figure is the latest sample, not an average");
});
test("a range is never longer than a year, and never shorter than a day", async () => {
const huge = await metrics.getServerMetrics(serverId, 9999, NOW);
assert.ok(huge.length > 0);
const tiny = await metrics.getServerMetrics(serverId, 0, NOW);
assert.ok(Array.isArray(tiny));
});
test("a slow, steady climb that two weeks call flat is seen over ninety days", async () => {
const other = (await db.insert(servers).values({ name: "slow", apiTokenHash: "s", apiTokenPrefix: "s", lastSeenAt: NOW.toISOString() }).returning({ id: servers.id }))[0].id;
// 0.07 a day is under what two weeks of data round to being worth a forecast once the noise is in, but over ninety days it is plainly a climb
for (let d = 90; d >= 1; d--) await metrics.recordServerMetrics(other, system(10, 40 + (90 - d) * 0.07 + (d % 2 ? 0.01 : -0.01)), ago(d));
const series = await metrics.getForecastSeries(other, NOW);
const disk = series.find((s) => s.metric === "disk:/")!;
assert.ok(disk.trend && disk.trend.daysToFull !== null, "there is a forecast");
assert.ok(disk.trend!.fit > 0.9);
});
test("the hourly and daily figures are kept so long and no longer", async () => {
const old = (await db.insert(servers).values({ name: "old", apiTokenHash: "o", apiTokenPrefix: "o" }).returning({ id: servers.id }))[0].id;
await db.insert(serverMetricsRollup).values([
{ serverId: old, metric: "cpu", grain: "h", at: ago(metrics.HOURLY_KEEP_DAYS + 3).toISOString(), sum: 1, n: 1, min: 1, max: 1 },
{ serverId: old, metric: "cpu", grain: "d", at: ago(metrics.HOURLY_KEEP_DAYS + 3).toISOString(), sum: 1, n: 1, min: 1, max: 1 },
{ serverId: old, metric: "cpu", grain: "d", at: ago(metrics.DAILY_KEEP_DAYS + 3).toISOString(), sum: 1, n: 1, min: 1, max: 1 },
]);
await metrics.recordServerMetrics(old, system(5, 5), NOW);
const left = await db.select().from(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, old), eq(serverMetricsRollup.metric, "cpu")));
assert.deepEqual(left.map((r) => r.grain).sort(), ["d", "d", "h"], "the old hour is gone, the old day within two years stays, the one beyond is gone, and today's two are there");
assert.equal(left.filter((r) => r.grain === "h").length, 1);
});
test("the first time, the hours and days are built from the samples already kept — once, and exactly", async () => {
const legacy = (await db.insert(servers).values({ name: "legacy", apiTokenHash: "l", apiTokenPrefix: "l" }).returning({ id: servers.id }))[0].id;
const base = new Date("2026-03-10T10:05:00.000Z").getTime();
await db.insert(serverMetrics).values([
{ serverId: legacy, metric: "cpu", value: 10, recordedAt: new Date(base).toISOString() },
{ serverId: legacy, metric: "cpu", value: 20, recordedAt: new Date(base + 10 * 60_000).toISOString() },
{ serverId: legacy, metric: "cpu", value: 60, recordedAt: new Date(base + HOUR).toISOString() },
]);
assert.equal(await metrics.backfillMetricRollups(), true);
assert.equal(await metrics.backfillMetricRollups(), false, "done once");
const rows = await db.select().from(serverMetricsRollup).where(eq(serverMetricsRollup.serverId, legacy));
const hours = rows.filter((r) => r.grain === "h").sort((a, b) => a.at.localeCompare(b.at));
assert.deepEqual(hours.map((r) => [r.at, r.n, r.sum, r.min, r.max]), [["2026-03-10T10:00:00.000Z", 2, 30, 10, 20], ["2026-03-10T11:00:00.000Z", 1, 60, 60, 60]]);
const days = rows.filter((r) => r.grain === "d");
assert.deepEqual(days.map((r) => [r.at, r.n, r.sum, r.min, r.max]), [["2026-03-10T00:00:00.000Z", 3, 90, 10, 60]]);
});
});
describe("how a check has done over a long time", () => {
const probe = (ok: boolean, ms: number | null) => async () => ({ ok, ms, message: ok ? "open" : "refused" });
const row = () => db.select().from(endpointChecks).where(eq(endpointChecks.id, checkId)).then((r) => r[0]);
test("each answer goes into its hour, and the average, the slowest and how many answered are kept", async () => {
const t = ago(5, -45 * 60_000); // five days ago, a little into an hour
await checks.runCheck(await row(), t, probe(true, 100));
await checks.runCheck(await row(), new Date(t.getTime() + 60_000), probe(true, 300));
await checks.runCheck(await row(), new Date(t.getTime() + 120_000), probe(false, null));
const hour = (await db.select().from(endpointHourly).where(eq(endpointHourly.checkId, checkId)))[0];
assert.deepEqual([hour.total, hour.ok, hour.msSum, hour.msCount, hour.msMax], [3, 2, 400, 2, 300]);
});
test("bucketing adds the figures up, so a slice is as true as the checks in it", () => {
const rows = [
{ t: 0, total: 10, ok: 10, msSum: 1000, msCount: 10, msMax: 400 },
{ t: 1000, total: 10, ok: 5, msSum: 1000, msCount: 5, msMax: 900 },
{ t: 9000, total: 4, ok: 4, msSum: 400, msCount: 4, msMax: 100 },
];
const points = checks.bucketHistory(rows, 0, 10_000, 2);
assert.deepEqual(points.map((p) => [p.availability, p.avgMs, p.maxMs, p.checks]), [[75, 133, 900, 20], [100, 100, 100, 4]]);
assert.deepEqual(checks.bucketHistory([], 0, 10, 2), []);
assert.equal(checks.bucketHistory([{ t: 0, total: 1, ok: 0, msSum: 0, msCount: 0, msMax: 0 }], 0, 10, 1)[0].avgMs, null, "no timed answer, no average");
});
test("a day is drawn from the results, a month from the hours, and half a year from the days", async () => {
// some results of today, kept in full
await db.insert(endpointResults).values([
{ checkId, at: ago(0, -30 * 60_000).toISOString(), ok: true, ms: 80, message: "open" },
{ checkId, at: ago(0, -20 * 60_000).toISOString(), ok: false, ms: null, message: "refused" },
]);
// and daily figures for the last 200 days
for (let d = 200; d >= 7; d--) {
const day = ago(d).toISOString().slice(0, 10);
await db.insert(endpointDaily).values({ checkId, day, total: 288, ok: d % 50 === 0 ? 250 : 288, msSum: 288 * 40, msCount: 288 }).onConflictDoNothing();
}
const day = await checks.getEndpointHistory(checkId, 1, NOW);
assert.equal(day.source, "results");
assert.deepEqual([day.checks, day.availability, day.avgMs], [2, 50, 80]);
const month = await checks.getEndpointHistory(checkId, 30, NOW);
assert.equal(month.source, "hourly");
assert.equal(month.checks, 3, "the three asked five days ago");
assert.equal(month.availability, 66.7);
assert.equal(month.points[0].maxMs, 300);
const half = await checks.getEndpointHistory(checkId, 200, NOW);
assert.equal(half.source, "daily");
assert.ok(half.points.length >= 150 && half.points.length <= 200, "a point a day");
assert.ok(half.availability !== null && half.availability < 100 && half.availability > 98);
assert.equal(half.avgMs, 40);
});
test("the first time, the hours are built from the results still kept — once", async () => {
const fresh = (await db.insert(endpointChecks).values({ name: "Other", kind: "tcp", target: "127.0.0.1", port: 2 }).returning({ id: endpointChecks.id }))[0].id;
await db.insert(endpointResults).values([
{ checkId: fresh, at: "2026-03-10T10:05:00.000Z", ok: true, ms: 50, message: "" },
{ checkId: fresh, at: "2026-03-10T10:35:00.000Z", ok: true, ms: 150, message: "" },
{ checkId: fresh, at: "2026-03-10T10:45:00.000Z", ok: false, ms: null, message: "" },
]);
assert.equal(await checks.backfillEndpointHourly(), true);
assert.equal(await checks.backfillEndpointHourly(), false);
const [hour] = await db.select().from(endpointHourly).where(eq(endpointHourly.checkId, fresh));
assert.deepEqual([hour.hour, hour.total, hour.ok, hour.msSum, hour.msCount, hour.msMax], ["2026-03-10T10:00:00.000Z", 3, 2, 200, 2, 150]);
});
test("the hours are pruned after four months", async () => {
await db.insert(endpointHourly).values({ checkId, hour: ago(checks.KEEP_HOURLY_DAYS + 5).toISOString(), total: 1, ok: 1, msSum: 1, msCount: 1, msMax: 1 });
await checks.pruneResults(NOW);
const left = await db.select().from(endpointHourly).where(eq(endpointHourly.checkId, checkId));
assert.ok(left.every((r) => Date.parse(r.hour) > NOW.getTime() - checks.KEEP_HOURLY_DAYS * DAY));
});
});
describe("over the routes", () => {
let app: Server;
let base = "";
before(async () => {
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "ada" } };
next();
});
a.use("/api/servers", serversRouter);
a.use("/api/endpoints", endpointsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => app.close());
test("a server's metrics can be asked for a year, and no more", async () => {
const year = (await (await fetch(`${base}/servers/${serverId}/metrics?days=365`)).json()) as any;
assert.equal(year.days, 365);
assert.ok(year.series.length > 0);
const more = (await (await fetch(`${base}/servers/${serverId}/metrics?days=5000`)).json()) as any;
assert.equal(more.days, 365);
const none = (await (await fetch(`${base}/servers/${serverId}/metrics`)).json()) as any;
assert.equal(none.days, 7);
});
test("a check's history: availability and answer times for a range, 404 for one that is not there", async () => {
const body = (await (await fetch(`${base}/endpoints/${checkId}/history?days=30`)).json()) as any;
assert.deepEqual([body.check.name, body.days, body.source, body.checks], ["Site", 30, "hourly", 5]); // the three asked five days ago, and the two results of today that the backfill above counted
assert.equal((await fetch(`${base}/endpoints/999999/history`)).status, 404);
assert.equal((await fetch(`${base}/endpoints/abc/history`)).status, 400);
assert.equal(((await (await fetch(`${base}/endpoints/${checkId}/history?days=99999`)).json()) as any).days, 365);
});
});
+279
View File
@@ -0,0 +1,279 @@
/**
* The read API and personal tokens: who may make a token, that the secret is shown once and only its hash kept, what a token can read (and that it can only
* read), and when it stops working.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-readapi-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
process.env.APP_BASE_URL = "https://hm.example.lan";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, assets, apiTokens, endpointChecks, endpointResults, alertIncidents, maintenanceWindows, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { createApp } = await import("../src/app.js");
const { apiTokensRouter } = await import("../src/routes/apiTokens.js");
const { setAck } = await import("../src/services/alertAcks.js");
const { invalidateAlertsCache } = await import("../src/services/alerts.js");
const { startOrExtendWindow } = await import("../src/services/maintenance.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const HOUR = 3600_000;
let app: Server;
let tokensApp: Server;
let base = "";
let tokensBase = "";
let who = "ada";
const ids: Record<string, number> = {};
const api = (path: string, secret: string | null, init: RequestInit = {}) =>
fetch(`${base}/api/v1${path}`, { ...init, headers: { ...(secret ? { authorization: `Bearer ${secret}` } : {}), ...(init.headers ?? {}) } });
const mine = (path: string, init: RequestInit = {}) => fetch(`${tokensBase}${path}`, { ...init, headers: { "content-type": "application/json", ...(init.headers ?? {}) } });
const make = async (label: string, extra: Record<string, unknown> = {}) => {
const res = await mine("", { method: "POST", body: JSON.stringify({ label, ...extra }) });
return { status: res.status, body: (await res.json()) as any };
};
before(async () => {
const rows = await db
.insert(users)
.values([
{ oidcSub: "ada", name: "Ada", role: "admin" },
{ oidcSub: "vic", name: "Vic", email: "vic@example.lan", role: "viewer" },
])
.returning({ id: users.id, oidcSub: users.oidcSub });
for (const r of rows) ids[r.oidcSub] = r.id;
// two servers (one reporting, one gone quiet), an alert on the quiet one, a piece of hardware, a check, an incident and a maintenance window
ids.web = (await db.insert(servers).values({ name: "web", apiTokenHash: "SECRET-HASH-WEB", apiTokenPrefix: "w", tags: JSON.stringify(["lab"]), ipAddresses: JSON.stringify(["10.0.0.5"]), lastSeenAt: new Date().toISOString(), cpuLoadPercent: 12, memTotalBytes: 1000, memUsedBytes: 400, disks: JSON.stringify([{ mount: "/", sizeBytes: 100, usedBytes: 40 }]) }).returning({ id: servers.id }))[0].id;
ids.db = (await db.insert(servers).values({ name: "db", apiTokenHash: "SECRET-HASH-DB", apiTokenPrefix: "d", lastSeenAt: new Date(Date.now() - 6 * HOUR).toISOString() }).returning({ id: servers.id }))[0].id;
await db.insert(assets).values({ name: "Rack switch", kind: "network", location: "Rack A", purchasePrice: 120, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() });
ids.check = (await db.insert(endpointChecks).values({ name: "Site", kind: "tcp", target: "127.0.0.1", port: 1, lastOk: true, lastCheckedAt: new Date().toISOString(), lastResponseMs: 12 }).returning({ id: endpointChecks.id }))[0].id;
await db.insert(endpointResults).values({ checkId: ids.check, at: new Date().toISOString(), ok: true, ms: 12, message: "open" });
await db.insert(alertIncidents).values({ alertKey: "offline:offline:server:1", severity: "critical", peakSeverity: "critical", category: "offline", source: "Server", message: "old", openedAt: new Date(Date.now() - 9 * HOUR).toISOString(), resolvedAt: new Date(Date.now() - 8 * HOUR).toISOString(), lastSeenAt: new Date().toISOString() });
await startOrExtendWindow("server", ids.web, 60, "Patching", "Ada");
await setAck({ alertId: `offline:offline:server:${ids.db}`, note: "On it", assigneeId: ids.ada, hours: 4, actor: "Ada" });
invalidateAlertsCache();
app = await new Promise<Server>((resolve) => {
const s = createApp().listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/", apiTokensRouter);
tokensApp = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
tokensBase = `http://127.0.0.1:${(tokensApp.address() as AddressInfo).port}`;
});
after(() => {
app.close();
tokensApp.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows may still hold the database file
}
});
describe("making and keeping tokens", () => {
let secret = "";
let tokenId = 0;
test("anyone signed in makes their own; the secret is in the answer once, and only a hash is kept", async () => {
who = "vic";
const made = await make("Home Assistant", { expiresInDays: 90 });
assert.equal(made.status, 201);
secret = made.body.secret;
tokenId = made.body.token.id;
assert.match(secret, /^hlm_p_[A-Za-z0-9_-]{20,}$/);
assert.ok(made.body.token.expiresAt, "it ends");
const stored = (await db.select().from(apiTokens).where(eq(apiTokens.id, tokenId)))[0];
assert.notEqual(stored.tokenHash, secret);
assert.equal(stored.tokenHash.length, 64);
assert.equal(stored.userId, ids.vic);
const list = await (await mine("")).json();
assert.deepEqual(list.tokens.map((t: any) => t.label), ["Home Assistant"]);
assert.ok(!JSON.stringify(list).includes(secret) && !JSON.stringify(list).includes(stored.tokenHash), "the list never has the secret or the hash");
});
test("a token needs a name, and the end is 30, 90 or 365 days or none", async () => {
assert.equal((await make("")).status, 400);
assert.equal((await make("x", { expiresInDays: 7 })).status, 400);
const forever = await make("No end");
assert.equal(forever.status, 201);
assert.equal(forever.body.token.expiresAt, null);
await mine(`/${forever.body.token.id}`, { method: "DELETE" });
});
test("a person has at most ten", async () => {
who = "ada";
for (let i = 0; i < 10; i++) assert.equal((await make(`t${i}`)).status, 201);
const eleventh = await make("one too many");
assert.equal(eleventh.status, 400);
assert.match(eleventh.body.message, /already 10/);
for (const t of (await (await mine("")).json()).tokens) await mine(`/${t.id}`, { method: "DELETE" });
});
test("somebody else's token is not yours to see or remove", async () => {
who = "ada";
assert.equal(((await (await mine("")).json()) as any).tokens.length, 0, "Ada sees none of Vic's");
assert.equal((await mine(`/${tokenId}`, { method: "DELETE" })).status, 404);
assert.equal((await api("/me", secret)).status, 200, "and it still works");
});
test("removing your own stops it at once", async () => {
who = "vic";
assert.equal((await api("/me", secret)).status, 200);
assert.equal((await mine(`/${tokenId}`, { method: "DELETE" })).status, 204);
assert.equal((await api("/me", secret)).status, 401);
const entries = (await db.select().from(auditLog)).filter((e) => e.category === "api_token");
assert.ok(entries.some((e) => e.action === "create") && entries.some((e) => e.action === "delete"), "making and removing a token is written down");
assert.ok(entries.every((e) => !JSON.stringify(e).includes("hlm_p_")), "and the secret is not");
});
});
describe("what a token can read", () => {
let secret = "";
before(async () => {
who = "vic";
secret = (await make("Reader")).body.secret;
});
test("no token, a wrong one, and one in the address are all the same 401", async () => {
for (const res of [await api("/servers", null), await api("/servers", "hlm_p_wrongwrongwrongwrongwrong"), await fetch(`${base}/api/v1/servers?token=${secret}`)]) {
assert.equal(res.status, 401);
assert.equal(res.headers.get("www-authenticate"), 'Bearer realm="homelab-manager"');
assert.equal(((await res.json()) as any).error, "unauthorized");
}
});
test("it only reads: any other verb is refused, even with a good token", async () => {
for (const method of ["POST", "PUT", "PATCH", "DELETE"]) {
const res = await api("/servers", secret, { method });
assert.equal(res.status, 405, method);
assert.equal(res.headers.get("allow"), "GET, HEAD");
}
});
test("who the token is", async () => {
const body = (await (await api("/me", secret)).json()) as any;
assert.deepEqual([body.user.name, body.user.role, body.token.label], ["Vic", "viewer", "Reader"]);
});
test("servers: a stable shape, no token hashes, the state worked out, filtered by tag and status", async () => {
const res = await api("/servers", secret);
assert.equal(res.headers.get("cache-control"), "no-store");
const text = await res.text();
assert.ok(!text.includes("SECRET-HASH") && !text.includes("apiToken"), "nothing of the servers' own tokens");
const body = JSON.parse(text) as any;
const web = body.servers.find((s: any) => s.name === "web");
assert.deepEqual([web.status, web.tags, web.ipAddresses, web.cpu.loadPercent, web.disks[0].usedPercent, web.url], ["online", ["lab"], ["10.0.0.5"], 12, 40, `https://hm.example.lan/servers/${ids.web}`]);
assert.equal(body.servers.find((s: any) => s.name === "db").status, "offline");
assert.deepEqual(((await (await api("/servers?tag=lab", secret)).json()) as any).servers.map((s: any) => s.name), ["web"]);
assert.deepEqual(((await (await api("/servers?status=offline", secret)).json()) as any).servers.map((s: any) => s.name), ["db"]);
assert.equal((await api(`/servers/${ids.web}`, secret)).status, 200);
assert.equal((await api("/servers/99999", secret)).status, 404);
assert.equal((await api("/servers/abc", secret)).status, 400);
});
test("a server's metrics, for up to a year", async () => {
const body = (await (await api(`/servers/${ids.web}/metrics?days=9999`, secret)).json()) as any;
assert.equal(body.days, 365);
assert.ok(Array.isArray(body.series));
});
test("alerts: what is wrong, with whether somebody is on it, filterable; addresses are whole", async () => {
const body = (await (await api("/alerts", secret)).json()) as any;
const offline = body.alerts.find((a: any) => a.id === `offline:offline:server:${ids.db}`);
assert.ok(offline, "the quiet server is raised");
assert.deepEqual([offline.severity, offline.acknowledgement.by, offline.acknowledgement.assignee, offline.acknowledgement.note], ["critical", "Ada", "Ada", "On it"]);
assert.equal(offline.url, `https://hm.example.lan/servers/${ids.db}`);
assert.equal(body.needingAttention.acknowledged, 1);
assert.equal(((await (await api("/alerts?acknowledged=false", secret)).json()) as any).alerts.some((a: any) => a.id === offline.id), false);
assert.equal(((await (await api("/alerts?severity=info", secret)).json()) as any).alerts.length, 0);
});
test("incidents, inventory, network, endpoints and maintenance", async () => {
const incidents = (await (await api("/incidents?status=resolved", secret)).json()) as any;
assert.equal(incidents.incidents.length, 1);
assert.equal(((await (await api(`/incidents/${incidents.incidents[0].id}`, secret)).json()) as any).incident?.id ?? incidents.incidents[0].id, incidents.incidents[0].id);
const open = ((await (await api("/incidents?status=open", secret)).json()) as any).incidents;
assert.deepEqual(open.map((i: any) => i.alertKey), [`offline:offline:server:${ids.db}`], "the quiet server's alert is an incident that is still open");
const inventory = (await (await api("/inventory", secret)).json()) as any;
assert.deepEqual([inventory.assets[0].name, inventory.assets[0].location, inventory.assets[0].purchasePrice], ["Rack switch", "Rack A", 120]);
const network = (await (await api("/network", secret)).json()) as any;
assert.deepEqual(Object.keys(network).sort(), ["devices", "networks", "serverLinks", "ssids"]);
const endpoints = (await (await api("/endpoints", secret)).json()) as any;
assert.deepEqual([endpoints.endpoints[0].name, endpoints.endpoints[0].status, endpoints.endpoints[0].lastResponseMs], ["Site", "up", 12]);
const history = (await (await api(`/endpoints/${ids.check}/history?days=1`, secret)).json()) as any;
assert.deepEqual([history.check.name, history.source, history.checks], ["Site", "results", 1]);
assert.equal((await api("/endpoints/99999/history", secret)).status, 404);
const maintenance = (await (await api("/maintenance", secret)).json()) as any;
assert.deepEqual([maintenance.active.length, maintenance.active[0].target.name, maintenance.active[0].reason], [1, "web", "Patching"]);
});
test("an address that is not there says so, and the index lists them", async () => {
const res = await api("/nothing", secret);
assert.equal(res.status, 404);
const index = (await (await api("/", secret)).json()) as any;
assert.ok(index.endpoints.some((e: string) => e.includes("/api/v1/alerts")));
});
});
describe("when a token stops working", () => {
test("it ends on the day it was made to; a removed person's tokens go with them", async () => {
who = "ada";
const { body } = await make("Short");
const secret = body.secret;
assert.equal((await api("/me", secret)).status, 200);
// the end comes: move it into the past
await db.update(apiTokens).set({ expiresAt: new Date(Date.now() - 1000).toISOString() }).where(eq(apiTokens.id, body.token.id));
assert.equal((await api("/me", secret)).status, 401, "expired");
const listed = ((await (await mine("")).json()) as any).tokens.find((t: any) => t.id === body.token.id);
assert.equal(listed.expired, true, "and the list says so");
const gone = (await db.insert(users).values({ oidcSub: "temp", name: "Temp", role: "viewer" }).returning({ id: users.id }))[0].id;
const { createApiToken } = await import("../src/services/apiTokens.js");
const t = await createApiToken({ userId: gone, label: "Leaver", expiresInDays: null });
assert.equal((await api("/me", t.token)).status, 200);
await db.delete(users).where(eq(users.id, gone));
assert.equal((await api("/me", t.token)).status, 401, "the person is gone, so is the token");
assert.equal((await db.select().from(apiTokens).where(eq(apiTokens.userId, gone))).length, 0);
});
test("use is noted, but not more than once a minute", async () => {
who = "ada";
const { body } = await make("Used");
assert.equal((await db.select().from(apiTokens).where(eq(apiTokens.id, body.token.id)))[0].lastUsedAt, null);
await api("/me", body.secret);
const first = (await db.select().from(apiTokens).where(eq(apiTokens.id, body.token.id)))[0].lastUsedAt;
assert.ok(first);
await api("/me", body.secret);
assert.equal((await db.select().from(apiTokens).where(eq(apiTokens.id, body.token.id)))[0].lastUsedAt, first);
});
});
+3 -3
View File
@@ -23,7 +23,7 @@ const { users, servers, serverMetrics } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { serversRouter } = await import("../src/routes/servers.js");
const { syncServerTasks } = await import("../src/services/taskSync.js");
const { samplesFromSystem, recordServerMetrics, bucketPoints, trendOf, getServerMetrics, MIN_GAP_MS, RETENTION_DAYS } = await import("../src/services/serverMetrics.js");
const { samplesFromSystem, recordServerMetrics, bucketPoints, trendOf, getServerMetrics, MIN_GAP_MS, RETENTION_DAYS, MAX_RANGE_DAYS } = await import("../src/services/serverMetrics.js");
await runMigrations();
@@ -189,8 +189,8 @@ describe("GET /api/servers/:id/metrics", () => {
test("anyone signed in can read it, and the number of days is kept within what is stored", async () => {
const res = await fetch(`${base}/${serverId}/metrics?days=9999`);
assert.equal(res.status, 200);
const body = await res.json();
assert.equal(body.days, RETENTION_DAYS);
assert.equal((await res.json()).days, MAX_RANGE_DAYS, "a year at most: the rest is kept as hours and days");
const body = await (await fetch(`${base}/${serverId}/metrics?days=${RETENTION_DAYS}`)).json();
assert.deepEqual(body.series.map((s: { kind: string }) => s.kind), ["cpu", "mem", "disk"]);
assert.equal((await fetch(`${base}/${serverId}/metrics`)).status, 200);
});