A year of history, one chart, and a read API with personal tokens
History: server load, memory and disks are also kept as hourly (four months) and daily (over two years) averages with lowest and highest; endpoint answers as hourly (four months) beside the daily figures. Charts ask for 24 hours to a year and are drawn from whichever is fine enough. A disk that two weeks call flat is looked at again over ninety days for the forecast. Built once from what is kept. One chart: TimeChart replaces the sparkline and the multi-line chart: hover and arrow-key readout, a lowest/highest band, gaps shown as gaps, the same ranges everywhere and the choice remembered. Used for server trends, comparing servers, a check's availability and answer times (Endpoints > History) and a server's availability. Read API: /api/v1 (servers, metrics, alerts, incidents, inventory, network, endpoints and their history, maintenance) with personal read-only tokens made under the user menu: shown once, hashed, 30/90/365 days or none, ten at most, removed with their owner. GET only. Documented in API.md. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5e26e77340
commit
68b41b5586
47 files changed
+12360
-247
No files matched your search
@@ -0,0 +1,24 @@
|
||||
CREATE TABLE `endpoint_hourly` (
|
||||
`check_id` integer NOT NULL,
|
||||
`hour` text NOT NULL,
|
||||
`total` integer DEFAULT 0 NOT NULL,
|
||||
`ok` integer DEFAULT 0 NOT NULL,
|
||||
`ms_sum` integer DEFAULT 0 NOT NULL,
|
||||
`ms_count` integer DEFAULT 0 NOT NULL,
|
||||
`ms_max` integer DEFAULT 0 NOT NULL,
|
||||
PRIMARY KEY(`check_id`, `hour`),
|
||||
FOREIGN KEY (`check_id`) REFERENCES `endpoint_checks`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE `server_metrics_rollup` (
|
||||
`server_id` integer NOT NULL,
|
||||
`metric` text NOT NULL,
|
||||
`grain` text NOT NULL,
|
||||
`at` text NOT NULL,
|
||||
`sum` real NOT NULL,
|
||||
`n` integer NOT NULL,
|
||||
`min` real NOT NULL,
|
||||
`max` real NOT NULL,
|
||||
PRIMARY KEY(`server_id`, `metric`, `grain`, `at`),
|
||||
FOREIGN KEY (`server_id`) REFERENCES `servers`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
@@ -0,0 +1,13 @@
|
||||
CREATE TABLE `api_tokens` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`user_id` integer NOT NULL,
|
||||
`label` text NOT NULL,
|
||||
`token_hash` text NOT NULL,
|
||||
`token_prefix` text NOT NULL,
|
||||
`created_at` text DEFAULT (current_timestamp) NOT NULL,
|
||||
`last_used_at` text,
|
||||
`expires_at` text,
|
||||
FOREIGN KEY (`user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX `api_tokens_token_hash_unique` ON `api_tokens` (`token_hash`);
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -323,6 +323,20 @@
|
||||
"when": 1792516305790,
|
||||
"tag": "0045_military_exodus",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 46,
|
||||
"version": "6",
|
||||
"when": 1792518508025,
|
||||
"tag": "0046_tough_living_tribunal",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 47,
|
||||
"version": "6",
|
||||
"when": 1792519188097,
|
||||
"tag": "0047_eager_dorian_gray",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -47,6 +47,8 @@ import { alertsRouter } from "./routes/alerts.js";
|
||||
import { mapRouter } from "./routes/map.js";
|
||||
import { networkRouter } from "./routes/network.js";
|
||||
import { myWorkRouter } from "./routes/myWork.js";
|
||||
import { apiTokensRouter } from "./routes/apiTokens.js";
|
||||
import { publicApiRouter } from "./routes/publicApi.js";
|
||||
import { generatorRouter } from "./routes/generator.js";
|
||||
import { alertTicketsRouter } from "./routes/alertTickets.js";
|
||||
import { avatarsRouter } from "./routes/avatars.js";
|
||||
@@ -83,6 +85,8 @@ export function createApp() {
|
||||
app.use("/api/agent/report", agentReportRouter);
|
||||
// A job reporting that it ran: a token in the address, no session, and no browser to be checked for where it came from.
|
||||
app.use("/api/heartbeat", heartbeatPingRouter);
|
||||
// The read API: a personal token in a header, GET only, no session and nothing for the same-origin check to protect.
|
||||
app.use("/api/v1", publicApiRouter);
|
||||
// Everything after this is reached with a session cookie, so a request that came from another origin is refused here.
|
||||
app.use(sameOriginGuard);
|
||||
app.use(express.json());
|
||||
@@ -114,6 +118,7 @@ export function createApp() {
|
||||
}
|
||||
|
||||
app.use("/auth", authRouter);
|
||||
app.use("/api/me/tokens", apiTokensRouter);
|
||||
app.use("/api/me", meRouter);
|
||||
app.use("/api/avatars", avatarsRouter);
|
||||
app.use("/api/heartbeats", heartbeatsRouter);
|
||||
|
||||
@@ -263,6 +263,26 @@ export const serverMetrics = sqliteTable(
|
||||
(table) => [index("server_metrics_lookup_idx").on(table.serverId, table.metric, table.recordedAt)],
|
||||
);
|
||||
|
||||
// The same figures, kept longer and coarser: one row per server, figure and hour (grain "h", kept a few months) or day (grain "d", kept for years), with what is
|
||||
// needed to draw them — the sum and count (so the average can be worked out as more samples arrive), and the lowest and highest. `at` is the start of the hour
|
||||
// or day, as ISO. Written as each sample is recorded; see services/serverMetrics.ts.
|
||||
export const serverMetricsRollup = sqliteTable(
|
||||
"server_metrics_rollup",
|
||||
{
|
||||
serverId: integer("server_id")
|
||||
.notNull()
|
||||
.references(() => servers.id, { onDelete: "cascade" }),
|
||||
metric: text("metric").notNull(),
|
||||
grain: text("grain").$type<"h" | "d">().notNull(),
|
||||
at: text("at").notNull(),
|
||||
sum: real("sum").notNull(),
|
||||
n: integer("n").notNull(),
|
||||
min: real("min").notNull(),
|
||||
max: real("max").notNull(),
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.serverId, t.metric, t.grain, t.at] })],
|
||||
);
|
||||
|
||||
// ─── Settings (key/value) ───────────────────────────────────────────────────
|
||||
|
||||
export const settings = sqliteTable("settings", {
|
||||
@@ -993,6 +1013,24 @@ export const endpointDaily = sqliteTable(
|
||||
(t) => [primaryKey({ columns: [t.checkId, t.day] })],
|
||||
);
|
||||
|
||||
// The same, by the hour: how many times a check was asked in that hour, how many answered, and the answer times — kept a few months, between the individual
|
||||
// results (two weeks) and the daily figures (years), so an answer-time chart can reach back beyond two weeks without a point for every check.
|
||||
export const endpointHourly = sqliteTable(
|
||||
"endpoint_hourly",
|
||||
{
|
||||
checkId: integer("check_id")
|
||||
.notNull()
|
||||
.references(() => endpointChecks.id, { onDelete: "cascade" }),
|
||||
hour: text("hour").notNull(), // ISO, the start of the hour (UTC)
|
||||
total: integer("total").notNull().default(0),
|
||||
ok: integer("ok").notNull().default(0),
|
||||
msSum: integer("ms_sum").notNull().default(0),
|
||||
msCount: integer("ms_count").notNull().default(0),
|
||||
msMax: integer("ms_max").notNull().default(0),
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.checkId, t.hour] })],
|
||||
);
|
||||
|
||||
// A certificate, domain registration or secret whose expiry moved later: it was renewed. Written when the app sees it happen, so it only goes back
|
||||
// as far as the version that began watching (services/renewals.ts).
|
||||
export const renewals = sqliteTable(
|
||||
@@ -1050,6 +1088,25 @@ export const metricsTokens = sqliteTable("metrics_tokens", {
|
||||
lastUsedAt: text("last_used_at"), // ISO
|
||||
});
|
||||
|
||||
// ─── Personal API tokens: read-only access to /api/v1 for scripts and Home Assistant ─────
|
||||
|
||||
// One person's own: made by them, shown to them once, and gone when they are. Only a hash of the secret is kept. A token only ever reads (the API has no other verbs),
|
||||
// and sees what any signed-in person sees.
|
||||
export const apiTokens = sqliteTable("api_tokens", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
userId: integer("user_id")
|
||||
.notNull()
|
||||
.references(() => users.id, { onDelete: "cascade" }),
|
||||
label: text("label").notNull(), // e.g. "Home Assistant"
|
||||
tokenHash: text("token_hash").notNull().unique(),
|
||||
tokenPrefix: text("token_prefix").notNull(),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
lastUsedAt: text("last_used_at"), // ISO
|
||||
expiresAt: text("expires_at"), // ISO; null = until removed
|
||||
});
|
||||
|
||||
// ─── Event webhooks: tell another system when something happens ─────────────
|
||||
|
||||
export const webhookEvents = ["alert.opened", "alert.resolved", "maintenance.started", "maintenance.ended"] as const;
|
||||
|
||||
@@ -25,6 +25,8 @@ import { initEndpointMonitor } from "./services/endpointMonitor.js";
|
||||
import { initPatchPlans } from "./services/patchPlans.js";
|
||||
import { initFlapMonitor } from "./services/flapMonitor.js";
|
||||
import { initUnifiSync } from "./services/unifiSync.js";
|
||||
import { backfillMetricRollups } from "./services/serverMetrics.js";
|
||||
import { backfillEndpointHourly } from "./services/endpointChecks.js";
|
||||
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
|
||||
import { initIncidentRecorder } from "./services/incidents.js";
|
||||
import { getAlerts } from "./services/alerts.js";
|
||||
@@ -69,6 +71,8 @@ initEndpointMonitor();
|
||||
initPatchPlans();
|
||||
initFlapMonitor();
|
||||
initUnifiSync();
|
||||
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
|
||||
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
|
||||
|
||||
createApp().listen(env.port, () => {
|
||||
console.log(`homelab-manager listening on port ${env.port}`);
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { Router } from "express";
|
||||
import { z } from "zod";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { createApiToken, EXPIRY_CHOICES, listApiTokens, MAX_TOKENS_PER_PERSON, publicApiToken, removeApiToken, TokenLimitError } from "../services/apiTokens.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
/** A person's own API tokens: anyone signed in makes, sees and removes theirs — never anybody else's. */
|
||||
export const apiTokensRouter = Router();
|
||||
apiTokensRouter.use(requireAuth);
|
||||
|
||||
const createSchema = z.object({
|
||||
label: z.string().trim().min(1).max(60),
|
||||
// Days until it stops working; null (or left out) for until it is removed.
|
||||
expiresInDays: z.union([z.literal(30), z.literal(90), z.literal(365)]).nullable().optional(),
|
||||
});
|
||||
|
||||
apiTokensRouter.get("/", asyncHandler(async (req, res) => {
|
||||
const rows = await listApiTokens(req.currentUser!.id);
|
||||
res.json({ tokens: rows.map((r) => publicApiToken(r)), max: MAX_TOKENS_PER_PERSON, expiryChoices: EXPIRY_CHOICES });
|
||||
}));
|
||||
|
||||
// The secret is in this answer and nowhere else, ever.
|
||||
apiTokensRouter.post("/", asyncHandler(async (req, res) => {
|
||||
const parsed = createSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give the token a name, like Home Assistant, and choose 30, 90 or 365 days or no end." });
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
const { row, token } = await createApiToken({ userId: actor.id, label: parsed.data.label, expiresInDays: parsed.data.expiresInDays ?? null });
|
||||
await recordAudit({ actor, category: "api_token", action: "create", targetType: "api_token", targetId: row.id, detail: { label: row.label, expiresAt: row.expiresAt } });
|
||||
res.status(201).json({ token: publicApiToken(row), secret: token });
|
||||
} catch (err) {
|
||||
if (err instanceof TokenLimitError) return res.status(400).json({ error: "too_many", message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
apiTokensRouter.delete("/:id", asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const actor = req.currentUser!;
|
||||
const removed = await removeApiToken(actor.id, id);
|
||||
if (!removed) return res.status(404).json({ error: "not_found" });
|
||||
await recordAudit({ actor, category: "api_token", action: "delete", targetType: "api_token", targetId: id, detail: { label: removed.label } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -6,7 +6,7 @@ import { endpointChecks, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { CHECK_KINDS, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
|
||||
import { CHECK_KINDS, getEndpointHistory, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
|
||||
import { loadDismissed, loadSuggestions, saveDismissed } from "../services/endpointSuggestions.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
@@ -63,6 +63,17 @@ endpointsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
});
|
||||
}));
|
||||
|
||||
// How one check has done over a day, a week, a month, three months or a year: availability and answer times, for the chart on its page.
|
||||
endpointsRouter.get("/:id/history", asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [check] = await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
|
||||
if (!check) return res.status(404).json({ error: "not_found" });
|
||||
const requested = Number(req.query.days);
|
||||
const days = Number.isFinite(requested) ? Math.min(365, Math.max(1, Math.round(requested))) : 7;
|
||||
res.json({ check, ...(await getEndpointHistory(id, days)) });
|
||||
}));
|
||||
|
||||
function valuesOf(d: z.infer<typeof checkSchema>) {
|
||||
return {
|
||||
name: d.name,
|
||||
|
||||
@@ -2,12 +2,11 @@ import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, networkDevices, networkSegments, networkServerLinks, servers } from "../db/schema.js";
|
||||
import { integrations } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { dependentsOf, loadGraph, needsOfNode, nodeKey } from "../services/dependencies.js";
|
||||
import { activeSubjects } from "../services/maintenance.js";
|
||||
import { syncAllUnifi, syncStatuses, syncUnifiIntegration } from "../services/unifiSync.js";
|
||||
import { buildNetworkOverview } from "../services/networkOverview.js";
|
||||
import { syncAllUnifi, syncUnifiIntegration } from "../services/unifiSync.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const networkRouter = Router();
|
||||
@@ -19,65 +18,7 @@ networkRouter.use(requireAuth);
|
||||
* names. Everyone signed in can see it (it is the same picture the dependency map and the alerts draw on); only starting a fresh read needs more.
|
||||
*/
|
||||
networkRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [intRows, devices, segments, statuses, graph, silenced, linkRows, serverRows] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "unifi")),
|
||||
db.select().from(networkDevices),
|
||||
db.select().from(networkSegments),
|
||||
syncStatuses(),
|
||||
loadGraph(),
|
||||
activeSubjects(),
|
||||
db.select().from(networkServerLinks),
|
||||
db.select({ id: servers.id, name: servers.name }).from(servers),
|
||||
]);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
const intName = new Map(intRows.map((i) => [i.id, i.name]));
|
||||
const byExternal = new Map(devices.map((d) => [`${d.integrationId}:${d.externalId}`, d]));
|
||||
// The servers plugged into each device, by what the controller says is connected (see unifiSync.matchClientsToServers).
|
||||
const plugged = new Map<number, { serverId: number; serverName: string; port: number | null; wired: boolean }[]>();
|
||||
const serverLinks: { serverId: number; serverName: string; deviceId: number; deviceName: string; port: number | null; wired: boolean }[] = [];
|
||||
for (const l of linkRows) {
|
||||
const device = byExternal.get(`${l.integrationId}:${l.deviceExternalId}`);
|
||||
const name = serverName.get(l.serverId);
|
||||
if (!device || !name) continue;
|
||||
plugged.set(device.id, [...(plugged.get(device.id) ?? []), { serverId: l.serverId, serverName: name, port: l.port, wired: l.wired }]);
|
||||
serverLinks.push({ serverId: l.serverId, serverName: name, deviceId: device.id, deviceName: device.name, port: l.port, wired: l.wired });
|
||||
}
|
||||
res.json({
|
||||
serverLinks,
|
||||
integrations: intRows.map((i) => ({ id: i.id, name: i.name, enabled: i.enabled, status: statuses[i.id] ?? null })),
|
||||
devices: devices
|
||||
.map((d) => {
|
||||
const key = nodeKey("device", d.id);
|
||||
const uplink = d.uplinkExternalId ? byExternal.get(`${d.integrationId}:${d.uplinkExternalId}`) : undefined;
|
||||
return {
|
||||
id: d.id,
|
||||
integrationId: d.integrationId,
|
||||
integrationName: intName.get(d.integrationId) ?? "UniFi",
|
||||
siteName: d.siteName,
|
||||
name: d.name,
|
||||
model: d.model,
|
||||
kind: d.kind,
|
||||
ip: d.ip,
|
||||
mac: d.mac,
|
||||
state: d.state,
|
||||
uptimeSeconds: d.uptimeSeconds,
|
||||
firmware: d.firmware,
|
||||
firmwareUpdatable: d.firmwareUpdatable,
|
||||
uplink: uplink ? { id: uplink.id, name: uplink.name } : null,
|
||||
offlineSince: d.offlineSince,
|
||||
lastOnlineAt: d.lastOnlineAt,
|
||||
updatedAt: d.updatedAt,
|
||||
// How many things depend on it (directly or through others), and how many it needs: what the dependency map says about it.
|
||||
dependents: dependentsOf(graph.edges, key).length,
|
||||
needs: needsOfNode(graph.edges, key).length,
|
||||
inMaintenance: silenced.has(key),
|
||||
servers: (plugged.get(d.id) ?? []).sort((a, b) => a.serverName.localeCompare(b.serverName)),
|
||||
};
|
||||
})
|
||||
.sort((a, b) => a.name.localeCompare(b.name)),
|
||||
networks: segments.filter((s) => s.kind === "network").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => (a.vlanId ?? 0) - (b.vlanId ?? 0) || a.name.localeCompare(b.name)),
|
||||
ssids: segments.filter((s) => s.kind === "ssid").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
});
|
||||
res.json(await buildNetworkOverview());
|
||||
}));
|
||||
|
||||
const syncSchema = z.object({ integrationId: z.number().int().positive().optional() });
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
import { Router, type NextFunction, type Request, type Response } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, servers } from "../db/schema.js";
|
||||
import { resolveApiToken, type ApiTokenRow, type ApiTokenUser } from "../services/apiTokens.js";
|
||||
import { getEndpointHistory } from "../services/endpointChecks.js";
|
||||
import { getIncident, listIncidents } from "../services/incidents.js";
|
||||
import { buildNetworkOverview } from "../services/networkOverview.js";
|
||||
import { alertsV1, endpointsV1, inventoryV1, maintenanceV1, oneServerV1, serversV1 } from "../services/publicApi.js";
|
||||
import { getServerMetrics, MAX_RANGE_DAYS } from "../services/serverMetrics.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { rateLimit } from "../utils/rateLimit.js";
|
||||
|
||||
/**
|
||||
* The read API: JSON about servers, alerts, incidents, inventory, network equipment, endpoint checks and maintenance, for scripts and things like Home Assistant.
|
||||
* A personal token in the Authorization header (never the address, which ends up in logs); GET only; and nothing here writes anything.
|
||||
*/
|
||||
export const publicApiRouter = Router();
|
||||
|
||||
declare module "express-serve-static-core" {
|
||||
interface Request {
|
||||
apiToken?: { token: ApiTokenRow; user: ApiTokenUser };
|
||||
}
|
||||
}
|
||||
|
||||
const limiter = rateLimit({ windowMs: 60_000, max: 300, message: "Too many requests." });
|
||||
|
||||
publicApiRouter.use(limiter);
|
||||
publicApiRouter.use((req: Request, res: Response, next: NextFunction) => {
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
if (req.method !== "GET" && req.method !== "HEAD") {
|
||||
res.setHeader("Allow", "GET, HEAD");
|
||||
return res.status(405).json({ error: "read_only", message: "This API only reads. It has no way to change anything." });
|
||||
}
|
||||
next();
|
||||
});
|
||||
|
||||
publicApiRouter.use(
|
||||
asyncHandler(async (req, res, next) => {
|
||||
const match = /^Bearer\s+(\S+)$/i.exec(req.get("authorization") ?? "");
|
||||
const found = match ? await resolveApiToken(match[1]) : null;
|
||||
if (!found) {
|
||||
// The same answer for no token, a wrong one, an expired one and a removed one.
|
||||
res.setHeader("WWW-Authenticate", 'Bearer realm="homelab-manager"');
|
||||
return res.status(401).json({ error: "unauthorized", message: "A token is needed: send it as 'Authorization: Bearer <token>'. Make one from the user menu → API tokens." });
|
||||
}
|
||||
req.apiToken = found;
|
||||
next();
|
||||
}),
|
||||
);
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
const daysOf = (v: unknown, fallback: number) => {
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? Math.min(MAX_RANGE_DAYS, Math.max(1, Math.round(n))) : fallback;
|
||||
};
|
||||
|
||||
publicApiRouter.get("/", (_req, res) => {
|
||||
res.json({
|
||||
name: "Homelab Manager read API",
|
||||
version: 1,
|
||||
endpoints: [
|
||||
"GET /api/v1/me",
|
||||
"GET /api/v1/servers?tag=&status=",
|
||||
"GET /api/v1/servers/:id",
|
||||
"GET /api/v1/servers/:id/metrics?days=",
|
||||
"GET /api/v1/alerts?severity=&category=&acknowledged=&silenced=",
|
||||
"GET /api/v1/incidents?status=&limit=",
|
||||
"GET /api/v1/incidents/:id",
|
||||
"GET /api/v1/inventory",
|
||||
"GET /api/v1/network",
|
||||
"GET /api/v1/endpoints",
|
||||
"GET /api/v1/endpoints/:id/history?days=",
|
||||
"GET /api/v1/maintenance",
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
publicApiRouter.get("/me", (req, res) => {
|
||||
const { token, user } = req.apiToken!;
|
||||
res.json({ user: { id: user.id, name: user.name, email: user.email, role: user.role }, token: { id: token.id, label: token.label, expiresAt: token.expiresAt } });
|
||||
});
|
||||
|
||||
publicApiRouter.get("/servers", asyncHandler(async (req, res) => {
|
||||
let list = await serversV1();
|
||||
if (typeof req.query.tag === "string" && req.query.tag) {
|
||||
const tag = req.query.tag.toLowerCase();
|
||||
list = list.filter((s) => s.tags.includes(tag));
|
||||
}
|
||||
if (req.query.status === "online" || req.query.status === "offline" || req.query.status === "unknown") list = list.filter((s) => s.status === req.query.status);
|
||||
res.json({ servers: list });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/servers/:id", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const server = await oneServerV1(id);
|
||||
if (!server) return res.status(404).json({ error: "not_found" });
|
||||
res.json({ server });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/servers/:id/metrics", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1);
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
const days = daysOf(req.query.days, 7);
|
||||
res.json({ days, series: await getServerMetrics(id, days) });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/alerts", asyncHandler(async (req, res) => {
|
||||
const report = await alertsV1();
|
||||
let alerts = report.alerts;
|
||||
const q = req.query;
|
||||
if (typeof q.severity === "string") alerts = alerts.filter((a) => a.severity === q.severity);
|
||||
if (typeof q.category === "string") alerts = alerts.filter((a) => a.category === q.category);
|
||||
if (q.acknowledged === "true" || q.acknowledged === "false") alerts = alerts.filter((a) => (a.acknowledgement !== null) === (q.acknowledged === "true"));
|
||||
if (q.silenced === "true" || q.silenced === "false") alerts = alerts.filter((a) => a.silenced === (q.silenced === "true"));
|
||||
res.json({ ...report, alerts });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/incidents", asyncHandler(async (req, res) => {
|
||||
const limit = Math.min(500, Math.max(1, Math.round(Number(req.query.limit)) || 100));
|
||||
let list = await listIncidents({ limit });
|
||||
if (req.query.status === "open") list = list.filter((i) => i.open);
|
||||
else if (req.query.status === "resolved") list = list.filter((i) => !i.open);
|
||||
res.json({ incidents: list });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/incidents/:id", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const detail = await getIncident(id);
|
||||
if (!detail) return res.status(404).json({ error: "not_found" });
|
||||
res.json(detail);
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/inventory", asyncHandler(async (_req, res) => {
|
||||
res.json({ assets: await inventoryV1() });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/network", asyncHandler(async (_req, res) => {
|
||||
const overview = await buildNetworkOverview();
|
||||
res.json({ devices: overview.devices, networks: overview.networks, ssids: overview.ssids, serverLinks: overview.serverLinks });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/endpoints", asyncHandler(async (_req, res) => {
|
||||
res.json({ endpoints: await endpointsV1() });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/endpoints/:id/history", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [check] = await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
|
||||
if (!check) return res.status(404).json({ error: "not_found" });
|
||||
res.json({ check, ...(await getEndpointHistory(id, daysOf(req.query.days, 7))) });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/maintenance", asyncHandler(async (_req, res) => {
|
||||
res.json(await maintenanceV1());
|
||||
}));
|
||||
|
||||
publicApiRouter.use((_req, res) => {
|
||||
res.status(404).json({ error: "not_found", message: "No such address. GET /api/v1 lists them." });
|
||||
});
|
||||
@@ -12,7 +12,7 @@ import { markIgnoredServices, parseHostHealth } from "../services/hostHealth.js"
|
||||
import { describeDiagnostics, parseDiagnostics } from "../services/agentDiagnostics.js";
|
||||
import { checkThresholds, parseThresholds } from "../services/serverThresholds.js";
|
||||
import { latestAgentVersions } from "../services/agentVersions.js";
|
||||
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
|
||||
import { getServerMetrics, MAX_RANGE_DAYS } from "../services/serverMetrics.js";
|
||||
import { getServerActivity } from "../services/serverActivity.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { ID_PATTERNS } from "../utils/pathSegment.js";
|
||||
@@ -272,7 +272,7 @@ serversRouter.get("/:id/metrics", asyncHandler(async (req, res) => {
|
||||
const [server] = await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1);
|
||||
if (!server) return res.status(404).json({ error: "not_found" });
|
||||
const requested = Number(req.query.days);
|
||||
const days = Number.isFinite(requested) ? Math.min(RETENTION_DAYS, Math.max(1, Math.round(requested))) : 7;
|
||||
const days = Number.isFinite(requested) ? Math.min(MAX_RANGE_DAYS, Math.max(1, Math.round(requested))) : 7;
|
||||
res.json({ days, series: await getServerMetrics(id, days) });
|
||||
}));
|
||||
|
||||
|
||||
@@ -6,6 +6,9 @@ import { getUptime } from "../services/uptimeHistory.js";
|
||||
export const uptimeRouter = Router();
|
||||
|
||||
// Anyone signed in who can see the server and integration lists can see how those have been doing.
|
||||
uptimeRouter.get("/", requireAuth, asyncHandler(async (_req, res) => {
|
||||
res.json(await getUptime());
|
||||
// `?days=` is 30 by default and at most a year (as far back as the audit log it is worked out from goes).
|
||||
uptimeRouter.get("/", requireAuth, asyncHandler(async (req, res) => {
|
||||
const requested = Number(req.query.days);
|
||||
const days = Number.isFinite(requested) ? Math.min(365, Math.max(1, Math.round(requested))) : undefined;
|
||||
res.json(await getUptime(new Date(), days));
|
||||
}));
|
||||
@@ -12,7 +12,7 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, heartbeats, integrations, networkDevices, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
import { recordFromReport } from "./incidents.js";
|
||||
@@ -499,7 +499,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const own = parseThresholds(s.thresholds);
|
||||
const forecastDays = effectiveThresholds(healthChecks, own).diskForecastDays;
|
||||
if (!(forecastDays > 0)) continue;
|
||||
const series = await getServerMetrics(s.id, 14);
|
||||
const series = await getForecastSeries(s.id);
|
||||
for (const f of disksFillingSoon(series, forecastDays, (mount) => diskLimit(own, mount, healthChecks.diskUsagePercent))) {
|
||||
push({
|
||||
category: "disk",
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* Personal API tokens: read-only access to /api/v1 for scripts, Home Assistant and the like. Each belongs to the person who made it — they see and remove their own —
|
||||
* and goes when they do. Like an agent token or a share link, the secret is shown once, when it is made, and only its SHA-256 is kept.
|
||||
*/
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { apiTokens, users } from "../db/schema.js";
|
||||
|
||||
export type ApiTokenRow = typeof apiTokens.$inferSelect;
|
||||
export type ApiTokenUser = typeof users.$inferSelect;
|
||||
|
||||
const hash = (token: string) => createHash("sha256").update(token).digest("hex");
|
||||
|
||||
/** Most tokens one person may have: one for each thing that reads, and a spare. */
|
||||
export const MAX_TOKENS_PER_PERSON = 10;
|
||||
/** How long a token may be asked to last, in days (or never). */
|
||||
export const EXPIRY_CHOICES = [30, 90, 365] as const;
|
||||
|
||||
/** Use is not recorded more than once in this time, so a script that reads every few seconds doesn't write the database each time. */
|
||||
const USED_WRITE_GAP_MS = 60_000;
|
||||
|
||||
export class TokenLimitError extends Error {}
|
||||
|
||||
export async function createApiToken(params: { userId: number; label: string; expiresInDays: number | null }, now: Date = new Date()): Promise<{ row: ApiTokenRow; token: string }> {
|
||||
const existing = await db.select({ id: apiTokens.id }).from(apiTokens).where(eq(apiTokens.userId, params.userId));
|
||||
if (existing.length >= MAX_TOKENS_PER_PERSON) throw new TokenLimitError(`There are already ${MAX_TOKENS_PER_PERSON} tokens. Remove one you don't use first.`);
|
||||
const token = `hlm_p_${randomBytes(24).toString("base64url")}`;
|
||||
const expiresAt = params.expiresInDays === null ? null : new Date(now.getTime() + params.expiresInDays * 86_400_000).toISOString();
|
||||
const [row] = await db.insert(apiTokens).values({ userId: params.userId, label: params.label, tokenHash: hash(token), tokenPrefix: token.slice(0, 12), expiresAt }).returning();
|
||||
return { row, token };
|
||||
}
|
||||
|
||||
/** The token a secret belongs to and whose it is, if it is one that still works; notes that it was used. */
|
||||
export async function resolveApiToken(secret: string, now: Date = new Date()): Promise<{ token: ApiTokenRow; user: ApiTokenUser } | null> {
|
||||
if (typeof secret !== "string" || secret.length < 20 || secret.length > 128) return null;
|
||||
const [found] = await db
|
||||
.select({ token: apiTokens, user: users })
|
||||
.from(apiTokens)
|
||||
.innerJoin(users, eq(users.id, apiTokens.userId))
|
||||
.where(eq(apiTokens.tokenHash, hash(secret)))
|
||||
.limit(1);
|
||||
if (!found) return null;
|
||||
if (found.token.expiresAt && Date.parse(found.token.expiresAt) <= now.getTime()) return null;
|
||||
if (!found.token.lastUsedAt || now.getTime() - Date.parse(found.token.lastUsedAt) >= USED_WRITE_GAP_MS) {
|
||||
await db.update(apiTokens).set({ lastUsedAt: now.toISOString() }).where(eq(apiTokens.id, found.token.id));
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/** A token as its owner's list shows it: everything but the hash. */
|
||||
export function publicApiToken(row: ApiTokenRow, now: Date = new Date()) {
|
||||
return {
|
||||
id: row.id,
|
||||
label: row.label,
|
||||
tokenPrefix: row.tokenPrefix,
|
||||
createdAt: row.createdAt,
|
||||
lastUsedAt: row.lastUsedAt,
|
||||
expiresAt: row.expiresAt,
|
||||
expired: row.expiresAt !== null && Date.parse(row.expiresAt) <= now.getTime(),
|
||||
};
|
||||
}
|
||||
|
||||
export async function listApiTokens(userId: number) {
|
||||
return db.select().from(apiTokens).where(eq(apiTokens.userId, userId)).orderBy(apiTokens.createdAt);
|
||||
}
|
||||
|
||||
/** Removes one of a person's own tokens. Returns whether there was one. */
|
||||
export async function removeApiToken(userId: number, id: number): Promise<ApiTokenRow | null> {
|
||||
const [row] = await db.select().from(apiTokens).where(and(eq(apiTokens.id, id), eq(apiTokens.userId, userId))).limit(1);
|
||||
if (!row) return null;
|
||||
await db.delete(apiTokens).where(eq(apiTokens.id, id));
|
||||
return row;
|
||||
}
|
||||
@@ -39,6 +39,7 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
|
||||
push: "settings",
|
||||
share_link: "settings",
|
||||
metrics_token: "settings",
|
||||
api_token: "settings",
|
||||
webhook: "settings",
|
||||
integration: "integrations",
|
||||
dns: "integrations",
|
||||
|
||||
@@ -14,7 +14,8 @@ import tls from "node:tls";
|
||||
import type { TLSSocket } from "node:tls";
|
||||
import { and, asc, eq, gte, inArray, lt, sql } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, endpointDaily, endpointResults } from "../db/schema.js";
|
||||
import { endpointChecks, endpointDaily, endpointHourly, endpointResults } from "../db/schema.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
import { noteRenewal } from "./renewals.js";
|
||||
import { resolveTarget } from "../utils/networkGuard.js";
|
||||
|
||||
@@ -246,6 +247,7 @@ export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c:
|
||||
.where(eq(endpointChecks.id, c.id));
|
||||
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
|
||||
await addToDaily(c.id, iso, result);
|
||||
await addToHourly(c.id, iso, result);
|
||||
// A certificate that is now good for much longer than it was has been renewed.
|
||||
if (result.certExpiresAt) await noteRenewal("certificate", c.name, c.certExpiresAt, result.certExpiresAt, now);
|
||||
return result;
|
||||
@@ -272,6 +274,45 @@ async function addToDaily(checkId: number, atIso: string, result: ProbeResult):
|
||||
}
|
||||
}
|
||||
|
||||
/** How long the hourly figures are kept: between the individual results (two weeks) and the daily figures (years). */
|
||||
export const KEEP_HOURLY_DAYS = 120;
|
||||
|
||||
/** Adds one result to its hour, so an answer-time chart can reach back beyond two weeks with a point an hour rather than one for every check. */
|
||||
async function addToHourly(checkId: number, atIso: string, result: ProbeResult): Promise<void> {
|
||||
const timed = result.ok && result.ms !== null;
|
||||
const ok = result.ok ? 1 : 0;
|
||||
const ms = timed ? (result.ms as number) : 0;
|
||||
const hour = `${atIso.slice(0, 13)}:00:00.000Z`;
|
||||
try {
|
||||
await db
|
||||
.insert(endpointHourly)
|
||||
.values({ checkId, hour, total: 1, ok, msSum: ms, msCount: timed ? 1 : 0, msMax: ms })
|
||||
.onConflictDoUpdate({
|
||||
target: [endpointHourly.checkId, endpointHourly.hour],
|
||||
set: { total: sql`${endpointHourly.total} + 1`, ok: sql`${endpointHourly.ok} + ${ok}`, msSum: sql`${endpointHourly.msSum} + ${ms}`, msCount: sql`${endpointHourly.msCount} + ${timed ? 1 : 0}`, msMax: sql`max(${endpointHourly.msMax}, ${ms})` },
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("[endpoints] couldn't add to the hourly figures:", err);
|
||||
}
|
||||
}
|
||||
|
||||
const HOURLY_BACKFILL_FLAG = "endpointHourlyBackfilled";
|
||||
|
||||
/** The first time after the hourly figures were added, builds them from the results still kept (two weeks). Done once. */
|
||||
export async function backfillEndpointHourly(): Promise<boolean> {
|
||||
if (await getInternalFlag(HOURLY_BACKFILL_FLAG)) return false;
|
||||
await db.run(sql`
|
||||
INSERT INTO endpoint_hourly (check_id, hour, total, ok, ms_sum, ms_count, ms_max)
|
||||
SELECT check_id, substr(at, 1, 13) || ':00:00.000Z', count(*), sum(ok), sum(CASE WHEN ok = 1 AND ms IS NOT NULL THEN ms ELSE 0 END),
|
||||
sum(CASE WHEN ok = 1 AND ms IS NOT NULL THEN 1 ELSE 0 END), coalesce(max(CASE WHEN ok = 1 AND ms IS NOT NULL THEN ms END), 0)
|
||||
FROM endpoint_results
|
||||
GROUP BY check_id, substr(at, 1, 13)
|
||||
ON CONFLICT (check_id, hour) DO UPDATE SET total = excluded.total, ok = excluded.ok, ms_sum = excluded.ms_sum, ms_count = excluded.ms_count, ms_max = excluded.ms_max
|
||||
`);
|
||||
await setInternalFlag(HOURLY_BACKFILL_FLAG, new Date().toISOString());
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
|
||||
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
|
||||
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
|
||||
@@ -290,6 +331,111 @@ export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRo
|
||||
export async function pruneResults(now: Date = new Date()): Promise<void> {
|
||||
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
|
||||
await db.delete(endpointDaily).where(lt(endpointDaily.day, new Date(now.getTime() - KEEP_DAILY_DAYS * 86_400_000).toISOString().slice(0, 10)));
|
||||
await db.delete(endpointHourly).where(lt(endpointHourly.hour, new Date(now.getTime() - KEEP_HOURLY_DAYS * 86_400_000).toISOString()));
|
||||
}
|
||||
|
||||
// ─── How a check has done over a long time ─────────────────────────────────
|
||||
|
||||
/** A row of figures for some stretch of time (one result, one hour or one day), with the moment it starts at as epoch ms. */
|
||||
export interface HistoryRow {
|
||||
t: number;
|
||||
total: number;
|
||||
ok: number;
|
||||
msSum: number;
|
||||
msCount: number;
|
||||
msMax: number;
|
||||
}
|
||||
|
||||
export interface HistoryPoint {
|
||||
/** ISO. */
|
||||
t: string;
|
||||
/** Share of the checks in this stretch that answered, 0–100. */
|
||||
availability: number | null;
|
||||
/** Average and longest answer time of the good, timed answers; null when there were none. */
|
||||
avgMs: number | null;
|
||||
maxMs: number | null;
|
||||
/** How many checks it stands for. */
|
||||
checks: number;
|
||||
}
|
||||
|
||||
/** Rows into at most `maxPoints` equal slices of [from, to]: figures are added up per slice, so a slice is as true as the checks in it. */
|
||||
export function bucketHistory(rows: HistoryRow[], from: number, to: number, maxPoints: number): HistoryPoint[] {
|
||||
if (rows.length === 0 || to <= from) return [];
|
||||
const size = Math.max((to - from) / maxPoints, 1);
|
||||
const buckets = new Map<number, { total: number; ok: number; msSum: number; msCount: number; msMax: number; tSum: number; rows: number }>();
|
||||
for (const r of rows) {
|
||||
if (r.t < from || r.t > to || r.total <= 0) continue;
|
||||
const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size));
|
||||
const b = buckets.get(key) ?? { total: 0, ok: 0, msSum: 0, msCount: 0, msMax: 0, tSum: 0, rows: 0 };
|
||||
b.total += r.total;
|
||||
b.ok += r.ok;
|
||||
b.msSum += r.msSum;
|
||||
b.msCount += r.msCount;
|
||||
b.msMax = Math.max(b.msMax, r.msMax);
|
||||
b.tSum += r.t;
|
||||
b.rows += 1;
|
||||
buckets.set(key, b);
|
||||
}
|
||||
return [...buckets.entries()]
|
||||
.sort((a, b) => a[0] - b[0])
|
||||
.map(([, b]) => ({
|
||||
t: new Date(b.tSum / b.rows).toISOString(),
|
||||
availability: Math.round((b.ok / b.total) * 1000) / 10,
|
||||
avgMs: b.msCount > 0 ? Math.round(b.msSum / b.msCount) : null,
|
||||
maxMs: b.msCount > 0 ? b.msMax : null,
|
||||
checks: b.total,
|
||||
}));
|
||||
}
|
||||
|
||||
export interface EndpointHistory {
|
||||
days: number;
|
||||
/** Which figures it was drawn from: the results themselves (a day or two), the hourly figures, or the daily ones. */
|
||||
source: "results" | "hourly" | "daily";
|
||||
points: HistoryPoint[];
|
||||
/** Over the whole stretch. */
|
||||
availability: number | null;
|
||||
avgMs: number | null;
|
||||
checks: number;
|
||||
}
|
||||
|
||||
/** How a check did over the last `days` (up to a year): availability and answer times, from the finest figures that reach back that far. */
|
||||
export async function getEndpointHistory(checkId: number, days: number, now: Date = new Date()): Promise<EndpointHistory> {
|
||||
days = Math.min(365, Math.max(1, Math.round(days)));
|
||||
const to = now.getTime();
|
||||
const from = to - days * 86_400_000;
|
||||
let rows: HistoryRow[];
|
||||
let source: EndpointHistory["source"];
|
||||
if (days <= 2) {
|
||||
source = "results";
|
||||
const found = await db
|
||||
.select({ at: endpointResults.at, ok: endpointResults.ok, ms: endpointResults.ms })
|
||||
.from(endpointResults)
|
||||
.where(and(eq(endpointResults.checkId, checkId), gte(endpointResults.at, new Date(from).toISOString())));
|
||||
rows = found.map((r) => {
|
||||
const timed = r.ok && r.ms !== null;
|
||||
return { t: Date.parse(r.at), total: 1, ok: r.ok ? 1 : 0, msSum: timed ? (r.ms as number) : 0, msCount: timed ? 1 : 0, msMax: timed ? (r.ms as number) : 0 };
|
||||
});
|
||||
} else if (days <= KEEP_HOURLY_DAYS) {
|
||||
source = "hourly";
|
||||
const found = await db.select().from(endpointHourly).where(and(eq(endpointHourly.checkId, checkId), gte(endpointHourly.hour, new Date(from).toISOString())));
|
||||
rows = found.map((r) => ({ t: Date.parse(r.hour), total: r.total, ok: r.ok, msSum: r.msSum, msCount: r.msCount, msMax: r.msMax }));
|
||||
} else {
|
||||
source = "daily";
|
||||
const found = await db.select().from(endpointDaily).where(and(eq(endpointDaily.checkId, checkId), gte(endpointDaily.day, new Date(from).toISOString().slice(0, 10))));
|
||||
// A day has no record of its slowest answer; its average stands in for it.
|
||||
rows = found.map((r) => ({ t: Date.parse(`${r.day}T00:00:00.000Z`), total: r.total, ok: r.ok, msSum: r.msSum, msCount: r.msCount, msMax: r.msCount > 0 ? Math.round(r.msSum / r.msCount) : 0 }));
|
||||
}
|
||||
const checks = rows.reduce((n, r) => n + r.total, 0);
|
||||
const okCount = rows.reduce((n, r) => n + r.ok, 0);
|
||||
const msCount = rows.reduce((n, r) => n + r.msCount, 0);
|
||||
return {
|
||||
days,
|
||||
source,
|
||||
points: bucketHistory(rows, from, to, days <= 2 ? 96 : days <= KEEP_HOURLY_DAYS ? 200 : 370),
|
||||
availability: checks === 0 ? null : Math.round((okCount / checks) * 1000) / 10,
|
||||
avgMs: msCount === 0 ? null : Math.round(rows.reduce((n, r) => n + r.msSum, 0) / msCount),
|
||||
checks,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── What the history says ─────────────────────────────────────────────────
|
||||
|
||||
@@ -19,7 +19,7 @@ import { getCachedBackupCoverage } from "./backupCoverage.js";
|
||||
import { lastAttempt as lastSelfBackup, listSnapshots } from "./selfBackup.js";
|
||||
import { loadAcks, attentionCounts } from "./alertAcks.js";
|
||||
import { listIncidents } from "./incidents.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
@@ -132,7 +132,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
|
||||
try {
|
||||
for (const s of serverRows) {
|
||||
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), 3650, 101)) {
|
||||
for (const f of disksFillingSoon(await getForecastSeries(s.id, now), 3650, 101)) {
|
||||
out.add("homelab_server_disk_days_to_full", "gauge", "Days until a disk is full at its present rate of filling; only for disks with a steady climb.", f.daysToFull, { server: s.name, mount: f.mount });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* The network equipment read from UniFi controllers: devices with their state, address, uptime and what they are connected through, the networks and the Wi-Fi
|
||||
* names, and which servers are plugged into which device. Shared by the Network devices page and the read API.
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, networkDevices, networkSegments, networkServerLinks, servers } from "../db/schema.js";
|
||||
import { dependentsOf, loadGraph, needsOfNode, nodeKey } from "./dependencies.js";
|
||||
import { activeSubjects } from "./maintenance.js";
|
||||
import { syncStatuses } from "./unifiSync.js";
|
||||
|
||||
export async function buildNetworkOverview() {
|
||||
const [intRows, devices, segments, statuses, graph, silenced, linkRows, serverRows] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "unifi")),
|
||||
db.select().from(networkDevices),
|
||||
db.select().from(networkSegments),
|
||||
syncStatuses(),
|
||||
loadGraph(),
|
||||
activeSubjects(),
|
||||
db.select().from(networkServerLinks),
|
||||
db.select({ id: servers.id, name: servers.name }).from(servers),
|
||||
]);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
const intName = new Map(intRows.map((i) => [i.id, i.name]));
|
||||
const byExternal = new Map(devices.map((d) => [`${d.integrationId}:${d.externalId}`, d]));
|
||||
// The servers plugged into each device, by what the controller says is connected (see unifiSync.matchClientsToServers).
|
||||
const plugged = new Map<number, { serverId: number; serverName: string; port: number | null; wired: boolean }[]>();
|
||||
const serverLinks: { serverId: number; serverName: string; deviceId: number; deviceName: string; port: number | null; wired: boolean }[] = [];
|
||||
for (const l of linkRows) {
|
||||
const device = byExternal.get(`${l.integrationId}:${l.deviceExternalId}`);
|
||||
const name = serverName.get(l.serverId);
|
||||
if (!device || !name) continue;
|
||||
plugged.set(device.id, [...(plugged.get(device.id) ?? []), { serverId: l.serverId, serverName: name, port: l.port, wired: l.wired }]);
|
||||
serverLinks.push({ serverId: l.serverId, serverName: name, deviceId: device.id, deviceName: device.name, port: l.port, wired: l.wired });
|
||||
}
|
||||
return {
|
||||
serverLinks,
|
||||
integrations: intRows.map((i) => ({ id: i.id, name: i.name, enabled: i.enabled, status: statuses[i.id] ?? null })),
|
||||
devices: devices
|
||||
.map((d) => {
|
||||
const key = nodeKey("device", d.id);
|
||||
const uplink = d.uplinkExternalId ? byExternal.get(`${d.integrationId}:${d.uplinkExternalId}`) : undefined;
|
||||
return {
|
||||
id: d.id,
|
||||
integrationId: d.integrationId,
|
||||
integrationName: intName.get(d.integrationId) ?? "UniFi",
|
||||
siteName: d.siteName,
|
||||
name: d.name,
|
||||
model: d.model,
|
||||
kind: d.kind,
|
||||
ip: d.ip,
|
||||
mac: d.mac,
|
||||
state: d.state,
|
||||
uptimeSeconds: d.uptimeSeconds,
|
||||
firmware: d.firmware,
|
||||
firmwareUpdatable: d.firmwareUpdatable,
|
||||
uplink: uplink ? { id: uplink.id, name: uplink.name } : null,
|
||||
offlineSince: d.offlineSince,
|
||||
lastOnlineAt: d.lastOnlineAt,
|
||||
updatedAt: d.updatedAt,
|
||||
// How many things depend on it (directly or through others), and how many it needs: what the dependency map says about it.
|
||||
dependents: dependentsOf(graph.edges, key).length,
|
||||
needs: needsOfNode(graph.edges, key).length,
|
||||
inMaintenance: silenced.has(key),
|
||||
servers: (plugged.get(d.id) ?? []).sort((a, b) => a.serverName.localeCompare(b.serverName)),
|
||||
};
|
||||
})
|
||||
.sort((a, b) => a.name.localeCompare(b.name)),
|
||||
networks: segments.filter((s) => s.kind === "network").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => (a.vlanId ?? 0) - (b.vlanId ?? 0) || a.name.localeCompare(b.name)),
|
||||
ssids: segments.filter((s) => s.kind === "ssid").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* What the read API (/api/v1) says. Each of these is a deliberate, stable shape — not the app's own page payloads, which change with the pages — and only things
|
||||
* any signed-in person can already see: no tokens, no secrets, no passwords, no settings. Dates are ISO 8601 in UTC; sizes are bytes; percentages are 0–100.
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, servers } from "../db/schema.js";
|
||||
import { env } from "../env.js";
|
||||
import { attentionCounts, loadAcks } from "./alertAcks.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { loadStats } from "./endpointChecks.js";
|
||||
import { describeTarget, listActiveWindows, listUpcomingWindows } from "./maintenance.js";
|
||||
import { parseHostHealth } from "./hostHealth.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { parseTags } from "./serverTags.js";
|
||||
import { parseThresholds } from "./serverThresholds.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { serverStatus } from "./topology.js";
|
||||
|
||||
/** An address inside the app as a full address, for a script that has no idea where the app lives. */
|
||||
export function absoluteUrl(path: string | null): string | null {
|
||||
return path === null ? null : `${env.appBaseUrl.replace(/\/+$/, "")}${path.startsWith("/") ? "" : "/"}${path}`;
|
||||
}
|
||||
|
||||
const json = <T>(text: string | null, fallback: T): T => {
|
||||
if (!text) return fallback;
|
||||
try {
|
||||
return JSON.parse(text) as T;
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
};
|
||||
|
||||
type ServerRow = typeof servers.$inferSelect;
|
||||
|
||||
export function serverV1(row: ServerRow, offlineMinutes: number, now: Date) {
|
||||
const status = serverStatus(row, offlineMinutes, now);
|
||||
const disks = json<{ mount: string; sizeBytes: number; usedBytes: number }[]>(row.disks, []);
|
||||
const updates = parsePendingUpdates(row.pendingUpdates);
|
||||
const health = parseHostHealth(row.hostHealth);
|
||||
return {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
hostname: row.hostname,
|
||||
description: row.description,
|
||||
os: row.osType,
|
||||
tags: parseTags(row.tags),
|
||||
ipAddresses: json<string[]>(row.ipAddresses, []),
|
||||
// "online" while its agent has reported within its limit, "offline" after, "unknown" if it never has.
|
||||
status: status === "ok" ? "online" : status === "bad" ? "offline" : "unknown",
|
||||
lastSeenAt: row.lastSeenAt,
|
||||
createdAt: row.createdAt,
|
||||
agentVersion: row.agentVersion ?? null,
|
||||
runsOnServerId: row.parentServerId ?? null,
|
||||
hardware: { manufacturer: row.hwManufacturer ?? null, model: row.hwModel ?? null, serial: row.hwSerial ?? null },
|
||||
cpu: { model: row.cpuModel, cores: row.cpuCores, loadPercent: row.cpuLoadPercent },
|
||||
memory: { totalBytes: row.memTotalBytes, usedBytes: row.memUsedBytes },
|
||||
disks: disks.map((d) => ({ mount: d.mount, sizeBytes: d.sizeBytes, usedBytes: d.usedBytes, usedPercent: d.sizeBytes > 0 ? Math.round((d.usedBytes / d.sizeBytes) * 1000) / 10 : null })),
|
||||
updates: updates ? { manager: updates.manager, count: updates.count, security: updates.security ?? null, rebootRequired: !!updates.rebootRequired, reportedAt: updates.reportedAt ?? null } : null,
|
||||
health: health ? { failedUnits: health.failedUnits?.length ?? 0, reportedAt: health.reportedAt ?? null } : null,
|
||||
thresholds: parseThresholds(row.thresholds),
|
||||
url: absoluteUrl(`/servers/${row.id}`),
|
||||
};
|
||||
}
|
||||
|
||||
export async function serversV1(now: Date = new Date()) {
|
||||
const [rows, settings] = await Promise.all([db.select().from(servers).orderBy(servers.name), getSettings()]);
|
||||
return rows.map((r) => serverV1(r, settings.healthChecks.serverOfflineMinutes, now));
|
||||
}
|
||||
|
||||
export async function oneServerV1(id: number, now: Date = new Date()) {
|
||||
const [row] = await db.select().from(servers).where(eq(servers.id, id)).limit(1);
|
||||
if (!row) return null;
|
||||
return serverV1(row, (await getSettings()).healthChecks.serverOfflineMinutes, now);
|
||||
}
|
||||
|
||||
export async function alertsV1() {
|
||||
const [report, acks] = await Promise.all([getAlerts(false), loadAcks()]);
|
||||
const attention = attentionCounts(report.alerts, acks);
|
||||
return {
|
||||
generatedAt: report.generatedAt,
|
||||
// What is on the page: those nobody has looked at, by severity, and how many somebody has.
|
||||
needingAttention: { critical: attention.critical, warning: attention.warning, acknowledged: attention.acknowledged },
|
||||
counts: report.counts,
|
||||
couldntCheck: report.couldntCheck,
|
||||
alerts: report.alerts.map((a) => {
|
||||
const ack = acks.get(a.id);
|
||||
return {
|
||||
id: a.id,
|
||||
severity: a.severity,
|
||||
category: a.category,
|
||||
source: a.source,
|
||||
message: a.message,
|
||||
url: absoluteUrl(a.link),
|
||||
silenced: a.silenced,
|
||||
flapping: !!a.flapping,
|
||||
target: a.target,
|
||||
causedBy: a.causedBy ?? null,
|
||||
affects: a.affects ?? [],
|
||||
acknowledgement: ack?.active ? { by: ack.ackedBy, at: ack.ackedAt, note: ack.note, until: ack.until, assignee: ack.assignee?.name ?? null } : null,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function inventoryV1() {
|
||||
const [rows, serverRows] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select({ id: servers.id, name: servers.name }).from(servers)]);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
return rows.map((a) => ({
|
||||
id: a.id,
|
||||
name: a.name,
|
||||
kind: a.kind,
|
||||
manufacturer: a.manufacturer,
|
||||
model: a.model,
|
||||
serial: a.serial,
|
||||
location: a.location,
|
||||
rackUnit: a.rackUnit,
|
||||
purchaseDate: a.purchaseDate,
|
||||
purchasePrice: a.purchasePrice,
|
||||
warrantyUntil: a.warrantyUntil,
|
||||
endOfSupport: a.endOfSupport,
|
||||
replacementCost: a.replacementCost,
|
||||
powerWatts: a.powerWatts,
|
||||
notes: a.notes,
|
||||
server: a.serverId !== null && serverName.has(a.serverId) ? { id: a.serverId, name: serverName.get(a.serverId)! } : null,
|
||||
updatedAt: a.updatedAt,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function maintenanceV1(now: Date = new Date()) {
|
||||
const shape = async (w: Awaited<ReturnType<typeof listActiveWindows>>[number]) => {
|
||||
const target = await describeTarget(w.targetType, w.targetId);
|
||||
return target ? { id: w.id, target: { type: w.targetType, id: w.targetId, name: target.name, kind: target.kind }, reason: w.reason, startsAt: w.startedAt, endsAt: w.endsAt, createdBy: w.createdBy } : null;
|
||||
};
|
||||
const active = (await Promise.all((await listActiveWindows(now)).map(shape))).filter((w): w is NonNullable<typeof w> => w !== null);
|
||||
const upcoming = (await Promise.all((await listUpcomingWindows(now)).map(shape))).filter((w): w is NonNullable<typeof w> => w !== null);
|
||||
return { active, upcoming };
|
||||
}
|
||||
|
||||
export async function endpointsV1(now: Date = new Date()) {
|
||||
const rows = await db.select().from(endpointChecks).orderBy(endpointChecks.name);
|
||||
const stats = await loadStats(rows.map((r) => r.id), now);
|
||||
return rows.map((r) => {
|
||||
const st = stats.get(r.id);
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
kind: r.kind,
|
||||
target: r.target,
|
||||
port: r.port,
|
||||
enabled: r.enabled,
|
||||
status: !r.enabled ? "paused" : r.lastOk === null ? "unknown" : r.lastOk ? "up" : "down",
|
||||
lastCheckedAt: r.lastCheckedAt,
|
||||
lastResponseMs: r.lastResponseMs,
|
||||
lastMessage: r.lastOk === false ? r.lastMessage : null,
|
||||
certificateExpiresAt: r.certExpiresAt,
|
||||
uptime24hPercent: st?.uptime24h ?? null,
|
||||
uptime7dPercent: st?.uptime7d ?? null,
|
||||
averageResponseMs24h: st?.avgMs24h ?? null,
|
||||
serverId: r.serverId,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -1,19 +1,30 @@
|
||||
/**
|
||||
* A short history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend
|
||||
* A history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend
|
||||
* (and say how soon a filling disk will run out) instead of only the latest figure.
|
||||
*
|
||||
* One sample is kept per server every ten minutes at most, however often its agent reports, for 35 days. Everything here is a
|
||||
* percentage: CPU load (as the agent reports it), memory in use, and each disk's used share.
|
||||
*
|
||||
* Beyond those 35 days the same figures are kept coarser: each sample also goes into its hour (kept four months) and its day (kept over two years), as a
|
||||
* sum, a count, a lowest and a highest. A chart asks for a number of days and is answered from whichever is fine enough and still there: the samples up to
|
||||
* 35 days, the hours up to 120, the days beyond. A year of a disk filling is then one point a day.
|
||||
*/
|
||||
import { and, desc, eq, gte, lt } from "drizzle-orm";
|
||||
import { and, desc, eq, gte, lt, sql } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { serverMetrics } from "../db/schema.js";
|
||||
import { serverMetrics, serverMetricsRollup } from "../db/schema.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
import type { IncomingSystemInfo } from "./taskSync.js";
|
||||
|
||||
/** Reports closer together than this add no sample. */
|
||||
export const MIN_GAP_MS = 10 * 60_000;
|
||||
export const RETENTION_DAYS = 35;
|
||||
/** How long the hourly and daily figures are kept. */
|
||||
export const HOURLY_KEEP_DAYS = 120;
|
||||
export const DAILY_KEEP_DAYS = 800;
|
||||
/** The longest stretch a chart may ask for. */
|
||||
export const MAX_RANGE_DAYS = 365;
|
||||
const DAY_MS = 24 * 3600_000;
|
||||
const HOUR_MS = 3600_000;
|
||||
|
||||
export interface Sample {
|
||||
metric: string;
|
||||
@@ -39,6 +50,35 @@ export function samplesFromSystem(system: IncomingSystemInfo | null | undefined)
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The start of the hour or day a moment falls in, as ISO. */
|
||||
export const hourStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / HOUR_MS) * HOUR_MS).toISOString();
|
||||
export const dayStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / DAY_MS) * DAY_MS).toISOString();
|
||||
|
||||
/** Adds samples to their hour and day. Never allowed to break recording the sample itself. */
|
||||
async function addToRollups(serverId: number, samples: Sample[], at: Date): Promise<void> {
|
||||
try {
|
||||
for (const grain of ["h", "d"] as const) {
|
||||
const bucket = grain === "h" ? hourStart(at.getTime()) : dayStart(at.getTime());
|
||||
for (const s of samples) {
|
||||
await db
|
||||
.insert(serverMetricsRollup)
|
||||
.values({ serverId, metric: s.metric, grain, at: bucket, sum: s.value, n: 1, min: s.value, max: s.value })
|
||||
.onConflictDoUpdate({
|
||||
target: [serverMetricsRollup.serverId, serverMetricsRollup.metric, serverMetricsRollup.grain, serverMetricsRollup.at],
|
||||
set: {
|
||||
sum: sql`${serverMetricsRollup.sum} + ${s.value}`,
|
||||
n: sql`${serverMetricsRollup.n} + 1`,
|
||||
min: sql`min(${serverMetricsRollup.min}, ${s.value})`,
|
||||
max: sql`max(${serverMetricsRollup.max}, ${s.value})`,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[metrics] couldn't add to the hourly and daily figures:", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Records a report's figures, unless this server was sampled a moment ago. Returns whether anything was stored. */
|
||||
export async function recordServerMetrics(serverId: number, system: IncomingSystemInfo | null | undefined, now: Date = new Date()): Promise<boolean> {
|
||||
const samples = samplesFromSystem(system);
|
||||
@@ -54,7 +94,32 @@ export async function recordServerMetrics(serverId: number, system: IncomingSyst
|
||||
|
||||
const at = now.toISOString();
|
||||
await db.insert(serverMetrics).values(samples.map((s) => ({ serverId, metric: s.metric, value: s.value, recordedAt: at })));
|
||||
await addToRollups(serverId, samples, now);
|
||||
await db.delete(serverMetrics).where(and(eq(serverMetrics.serverId, serverId), lt(serverMetrics.recordedAt, new Date(now.getTime() - RETENTION_DAYS * DAY_MS).toISOString())));
|
||||
await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "h"), lt(serverMetricsRollup.at, new Date(now.getTime() - HOURLY_KEEP_DAYS * DAY_MS).toISOString())));
|
||||
await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "d"), lt(serverMetricsRollup.at, new Date(now.getTime() - DAILY_KEEP_DAYS * DAY_MS).toISOString())));
|
||||
return true;
|
||||
}
|
||||
|
||||
const BACKFILL_FLAG = "metricsRollupBackfilled";
|
||||
|
||||
/**
|
||||
* The first time after the hourly and daily figures were added, builds them from the samples already kept (up to 35 days), so the longer ranges are not empty
|
||||
* for weeks. A bucket is written from what the samples say, replacing what is there, so it is right however far it got before. Done once.
|
||||
*/
|
||||
export async function backfillMetricRollups(): Promise<boolean> {
|
||||
if (await getInternalFlag(BACKFILL_FLAG)) return false;
|
||||
for (const [grain, len] of [["h", 13], ["d", 10]] as const) {
|
||||
const suffix = grain === "h" ? ":00:00.000Z" : "T00:00:00.000Z";
|
||||
await db.run(sql`
|
||||
INSERT INTO server_metrics_rollup (server_id, metric, grain, at, sum, n, min, max)
|
||||
SELECT server_id, metric, ${grain}, substr(recorded_at, 1, ${len}) || ${suffix}, sum(value), count(*), min(value), max(value)
|
||||
FROM server_metrics
|
||||
GROUP BY server_id, metric, substr(recorded_at, 1, ${len})
|
||||
ON CONFLICT (server_id, metric, grain, at) DO UPDATE SET sum = excluded.sum, n = excluded.n, min = excluded.min, max = excluded.max
|
||||
`);
|
||||
}
|
||||
await setInternalFlag(BACKFILL_FLAG, new Date().toISOString());
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -62,6 +127,9 @@ export interface Point {
|
||||
/** ISO time. */
|
||||
t: string;
|
||||
v: number;
|
||||
/** The lowest and highest figure in what this point stands for, when it stands for several (hourly and daily figures). */
|
||||
lo?: number;
|
||||
hi?: number;
|
||||
}
|
||||
|
||||
/** Averages points into at most `maxPoints` equal slices of [from, to], so a month of samples draws as a short line. */
|
||||
@@ -83,6 +151,42 @@ export function bucketPoints(points: { t: number; v: number }[], from: number, t
|
||||
.map(([, b]) => ({ t: new Date(b.tSum / b.n).toISOString(), v: round1(b.sum / b.n) }));
|
||||
}
|
||||
|
||||
/** A row of the hourly or daily figures, with the moment it starts at as epoch ms. */
|
||||
export interface RollupRow {
|
||||
t: number;
|
||||
sum: number;
|
||||
n: number;
|
||||
min: number;
|
||||
max: number;
|
||||
}
|
||||
|
||||
/** The average of what each row stands for. */
|
||||
export function rollupAverages(rows: RollupRow[]): { t: number; v: number }[] {
|
||||
return rows.filter((r) => r.n > 0).map((r) => ({ t: r.t, v: r.sum / r.n }));
|
||||
}
|
||||
|
||||
/** Rows into at most `maxPoints` slices of [from, to]: the average of the samples in each (not of the rows), with the lowest and highest. */
|
||||
export function bucketRollups(rows: RollupRow[], from: number, to: number, maxPoints: number): Point[] {
|
||||
if (rows.length === 0 || to <= from) return [];
|
||||
const size = Math.max((to - from) / maxPoints, 1);
|
||||
const buckets = new Map<number, { sum: number; n: number; lo: number; hi: number; tSum: number; rows: number }>();
|
||||
for (const r of rows) {
|
||||
if (r.t < from || r.t > to || r.n <= 0) continue;
|
||||
const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size));
|
||||
const b = buckets.get(key) ?? { sum: 0, n: 0, lo: Infinity, hi: -Infinity, tSum: 0, rows: 0 };
|
||||
b.sum += r.sum;
|
||||
b.n += r.n;
|
||||
b.lo = Math.min(b.lo, r.min);
|
||||
b.hi = Math.max(b.hi, r.max);
|
||||
b.tSum += r.t;
|
||||
b.rows += 1;
|
||||
buckets.set(key, b);
|
||||
}
|
||||
return [...buckets.entries()]
|
||||
.sort((a, b) => a[0] - b[0])
|
||||
.map(([, b]) => ({ t: new Date(b.tSum / b.rows).toISOString(), v: round1(b.sum / b.n), lo: round1(b.lo), hi: round1(b.hi) }));
|
||||
}
|
||||
|
||||
export interface Trend {
|
||||
/** Change in the percentage per day, from a straight line through the points. */
|
||||
perDay: number;
|
||||
@@ -175,35 +279,86 @@ function describe(metric: string): { kind: MetricSeries["kind"]; label: string }
|
||||
|
||||
const ORDER = { cpu: 0, mem: 1, disk: 2 } as const;
|
||||
|
||||
/** Every series recorded for a server over the last `days`, ready to draw. */
|
||||
/** Which of the kept figures answers a stretch of `days`: the samples while they reach back that far, then the hours, then the days. */
|
||||
export function sourceFor(days: number): "samples" | "hourly" | "daily" {
|
||||
return days <= RETENTION_DAYS ? "samples" : days <= HOURLY_KEEP_DAYS ? "hourly" : "daily";
|
||||
}
|
||||
|
||||
/** How many points a chart over `days` is drawn from at most. */
|
||||
const pointsFor = (days: number) => (days <= RETENTION_DAYS ? 96 : days <= HOURLY_KEEP_DAYS ? 200 : MAX_RANGE_DAYS + 5);
|
||||
|
||||
/** The latest sample of each figure of a server, whatever range is asked for. */
|
||||
async function latestSamples(serverId: number): Promise<Map<string, number>> {
|
||||
const rows = await db.all<{ metric: string; value: number }>(
|
||||
sql`SELECT metric, value FROM server_metrics WHERE id IN (SELECT max(id) FROM server_metrics WHERE server_id = ${serverId} GROUP BY metric)`,
|
||||
);
|
||||
return new Map(rows.map((r) => [r.metric, r.value]));
|
||||
}
|
||||
|
||||
/** Every series recorded for a server over the last `days` (up to a year), ready to draw. */
|
||||
export async function getServerMetrics(serverId: number, days: number, now: Date = new Date()): Promise<MetricSeries[]> {
|
||||
days = Math.min(MAX_RANGE_DAYS, Math.max(1, days));
|
||||
const to = now.getTime();
|
||||
const from = to - days * DAY_MS;
|
||||
const rows = await db
|
||||
.select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt })
|
||||
.from(serverMetrics)
|
||||
.where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString())))
|
||||
.orderBy(serverMetrics.recordedAt);
|
||||
const source = sourceFor(days);
|
||||
|
||||
const byMetric = new Map<string, { t: number; v: number }[]>();
|
||||
for (const r of rows) {
|
||||
const list = byMetric.get(r.metric) ?? [];
|
||||
list.push({ t: Date.parse(r.at), v: r.value });
|
||||
byMetric.set(r.metric, list);
|
||||
// The figures, per metric, as { t, v } for a trend and as points for the chart.
|
||||
const raw = new Map<string, { t: number; v: number }[]>();
|
||||
const points = new Map<string, Point[]>();
|
||||
if (source === "samples") {
|
||||
const rows = await db
|
||||
.select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt })
|
||||
.from(serverMetrics)
|
||||
.where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString())))
|
||||
.orderBy(serverMetrics.recordedAt);
|
||||
for (const r of rows) raw.set(r.metric, [...(raw.get(r.metric) ?? []), { t: Date.parse(r.at), v: r.value }]);
|
||||
for (const [metric, list] of raw) points.set(metric, bucketPoints(list, from, to, pointsFor(days)));
|
||||
} else {
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(serverMetricsRollup)
|
||||
.where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, source === "hourly" ? "h" : "d"), gte(serverMetricsRollup.at, new Date(from).toISOString())))
|
||||
.orderBy(serverMetricsRollup.at);
|
||||
const byMetric = new Map<string, RollupRow[]>();
|
||||
for (const r of rows) byMetric.set(r.metric, [...(byMetric.get(r.metric) ?? []), { t: Date.parse(r.at), sum: r.sum, n: r.n, min: r.min, max: r.max }]);
|
||||
for (const [metric, list] of byMetric) {
|
||||
raw.set(metric, rollupAverages(list));
|
||||
points.set(metric, bucketRollups(list, from, to, pointsFor(days)));
|
||||
}
|
||||
}
|
||||
const latest = source === "samples" ? null : await latestSamples(serverId);
|
||||
|
||||
return [...byMetric.entries()]
|
||||
.map(([metric, raw]) => {
|
||||
return [...raw.entries()]
|
||||
.map(([metric, list]) => {
|
||||
const { kind, label } = describe(metric);
|
||||
const newest = latest?.get(metric) ?? list[list.length - 1]?.v;
|
||||
return {
|
||||
metric,
|
||||
kind,
|
||||
label,
|
||||
points: bucketPoints(raw, from, to, 96),
|
||||
latest: raw[raw.length - 1]?.v ?? null,
|
||||
points: points.get(metric) ?? [],
|
||||
latest: newest === undefined ? null : source === "samples" ? newest : round1(newest),
|
||||
// A disk filling is the thing worth forecasting; load and memory go up and down by the hour.
|
||||
trend: kind === "disk" ? trendOf(raw) : null,
|
||||
trend: kind === "disk" ? trendOf(list) : null,
|
||||
};
|
||||
})
|
||||
.sort((a, b) => ORDER[a.kind] - ORDER[b.kind] || a.label.localeCompare(b.label));
|
||||
}
|
||||
|
||||
/** How far back the longer look at a disk goes when the last two weeks say it is flat. */
|
||||
export const LONG_FORECAST_DAYS = 90;
|
||||
|
||||
/**
|
||||
* The series to forecast from: the last two weeks, as before — except that a disk those two weeks call flat is looked at again over the last
|
||||
* ninety days, so a slow, steady climb (a point a week) is seen too. The longer trend replaces the short one only if it is really rising.
|
||||
*/
|
||||
export async function getForecastSeries(serverId: number, now: Date = new Date()): Promise<MetricSeries[]> {
|
||||
const short = await getServerMetrics(serverId, 14, now);
|
||||
if (!short.some((s) => s.kind === "disk" && (!s.trend || s.trend.daysToFull === null))) return short;
|
||||
const long = await getServerMetrics(serverId, LONG_FORECAST_DAYS, now);
|
||||
return short.map((s) => {
|
||||
if (s.kind !== "disk" || (s.trend && s.trend.daysToFull !== null)) return s;
|
||||
const longer = long.find((l) => l.metric === s.metric);
|
||||
return longer?.trend && longer.trend.daysToFull !== null ? { ...s, trend: longer.trend } : s;
|
||||
});
|
||||
}
|
||||
@@ -16,7 +16,7 @@ import { describeTarget, listUpcomingWindows } from "./maintenance.js";
|
||||
import { notify, notifyNow } from "./notify.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
@@ -163,7 +163,7 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
const own = parseThresholds(s.thresholds);
|
||||
const forecastDays = effectiveThresholds(settings.healthChecks, own).diskForecastDays;
|
||||
if (!(forecastDays > 0)) continue;
|
||||
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
|
||||
for (const f of disksFillingSoon(await getForecastSeries(s.id, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
|
||||
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* Longer history: server figures and endpoint answer times kept by the hour and the day beyond the days they are kept in full, and drawn from whichever is
|
||||
* fine enough for the range asked for, up to a year.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-history-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, serverMetrics, serverMetricsRollup, endpointChecks, endpointResults, endpointHourly, endpointDaily } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const metrics = await import("../src/services/serverMetrics.js");
|
||||
const checks = await import("../src/services/endpointChecks.js");
|
||||
const { serversRouter } = await import("../src/routes/servers.js");
|
||||
const { endpointsRouter } = await import("../src/routes/endpoints.js");
|
||||
const { eq, and } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const DAY = 24 * 3600_000;
|
||||
const HOUR = 3600_000;
|
||||
const NOW = new Date(Math.floor(Date.now() / HOUR) * HOUR + 20 * 60_000); // twenty minutes into an hour
|
||||
const ago = (days: number, extraMs = 0) => new Date(NOW.getTime() - days * DAY + extraMs);
|
||||
const system = (cpu: number, disk: number) => ({ cpu: { load_percent: cpu }, memory: { total_bytes: 1000, used_bytes: 500 }, disks: [{ mount: "/", size_bytes: 100, used_bytes: disk }] }) as never;
|
||||
|
||||
let serverId = 0;
|
||||
let checkId = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "ada", name: "Ada", role: "admin" }]);
|
||||
serverId = (await db.insert(servers).values({ name: "web", apiTokenHash: "w", apiTokenPrefix: "w", lastSeenAt: NOW.toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
checkId = (await db.insert(endpointChecks).values({ name: "Site", kind: "tcp", target: "127.0.0.1", port: 1 }).returning({ id: endpointChecks.id }))[0].id;
|
||||
});
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows may still hold the database file
|
||||
}
|
||||
});
|
||||
|
||||
describe("the hourly and daily figures of a server", () => {
|
||||
test("which figures answer which range: the samples while they reach, then the hours, then the days", () => {
|
||||
assert.deepEqual([1, 7, 35, 36, 120, 121, 365].map(metrics.sourceFor), ["samples", "samples", "samples", "hourly", "hourly", "daily", "daily"]);
|
||||
});
|
||||
|
||||
test("a sample goes into its hour and its day, and the next one in the same hour is added to it, with the lowest and highest kept", async () => {
|
||||
const t = new Date(ago(0).getTime() - 10 * 60_000); // ten minutes into the hour
|
||||
assert.equal(await metrics.recordServerMetrics(serverId, system(10, 50), t), true);
|
||||
assert.equal(await metrics.recordServerMetrics(serverId, system(30, 52), new Date(t.getTime() + 11 * 60_000)), true);
|
||||
const cpuHour = (await db.select().from(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.metric, "cpu"), eq(serverMetricsRollup.grain, "h"))))[0];
|
||||
assert.deepEqual([cpuHour.n, cpuHour.sum, cpuHour.min, cpuHour.max, cpuHour.at], [2, 40, 10, 30, metrics.hourStart(t.getTime())]);
|
||||
const cpuDay = (await db.select().from(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.metric, "cpu"), eq(serverMetricsRollup.grain, "d"))))[0];
|
||||
assert.deepEqual([cpuDay.n, cpuDay.at], [2, metrics.dayStart(t.getTime())]);
|
||||
});
|
||||
|
||||
test("a hundred days of a disk filling: seven days come from the samples, ninety from the hours, a year from the days", async () => {
|
||||
// one sample a day, climbing 0.3 points a day, oldest first (the way agents report)
|
||||
const hist = (await db.insert(servers).values({ name: "hist", apiTokenHash: "h", apiTokenPrefix: "h", lastSeenAt: NOW.toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
for (let d = 100; d >= 1; d--) await metrics.recordServerMetrics(hist, system(20, 30 + (100 - d) * 0.3), ago(d));
|
||||
const week = await metrics.getServerMetrics(hist, 7, NOW);
|
||||
const disk7 = week.find((s) => s.metric === "disk:/")!;
|
||||
assert.ok(disk7.points.length > 0 && disk7.points.every((p) => p.lo === undefined), "from the samples: no band");
|
||||
|
||||
const ninety = await metrics.getServerMetrics(hist, 90, NOW);
|
||||
const disk90 = ninety.find((s) => s.metric === "disk:/")!;
|
||||
assert.ok(disk90.points.length >= 85, `about a point a day (got ${disk90.points.length})`);
|
||||
assert.ok(disk90.points.every((p) => p.lo !== undefined && p.hi !== undefined && p.lo <= p.v && p.v <= p.hi), "from the hours: each has a lowest and highest");
|
||||
assert.ok(disk90.trend && Math.abs(disk90.trend.perDay - 0.3) < 0.05, "and the trend reaches back all ninety days");
|
||||
assert.ok(disk90.trend!.spanDays > 85);
|
||||
|
||||
const year = await metrics.getServerMetrics(hist, 365, NOW);
|
||||
const diskYear = year.find((s) => s.metric === "disk:/")!;
|
||||
assert.ok(diskYear.points.length >= 99 && diskYear.points.length <= 370);
|
||||
assert.equal(diskYear.latest, 30 + 99 * 0.3 > 100 ? 100 : Math.round((30 + 99 * 0.3) * 10) / 10, "the latest figure is the latest sample, not an average");
|
||||
});
|
||||
|
||||
test("a range is never longer than a year, and never shorter than a day", async () => {
|
||||
const huge = await metrics.getServerMetrics(serverId, 9999, NOW);
|
||||
assert.ok(huge.length > 0);
|
||||
const tiny = await metrics.getServerMetrics(serverId, 0, NOW);
|
||||
assert.ok(Array.isArray(tiny));
|
||||
});
|
||||
|
||||
test("a slow, steady climb that two weeks call flat is seen over ninety days", async () => {
|
||||
const other = (await db.insert(servers).values({ name: "slow", apiTokenHash: "s", apiTokenPrefix: "s", lastSeenAt: NOW.toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
// 0.07 a day is under what two weeks of data round to being worth a forecast once the noise is in, but over ninety days it is plainly a climb
|
||||
for (let d = 90; d >= 1; d--) await metrics.recordServerMetrics(other, system(10, 40 + (90 - d) * 0.07 + (d % 2 ? 0.01 : -0.01)), ago(d));
|
||||
const series = await metrics.getForecastSeries(other, NOW);
|
||||
const disk = series.find((s) => s.metric === "disk:/")!;
|
||||
assert.ok(disk.trend && disk.trend.daysToFull !== null, "there is a forecast");
|
||||
assert.ok(disk.trend!.fit > 0.9);
|
||||
});
|
||||
|
||||
test("the hourly and daily figures are kept so long and no longer", async () => {
|
||||
const old = (await db.insert(servers).values({ name: "old", apiTokenHash: "o", apiTokenPrefix: "o" }).returning({ id: servers.id }))[0].id;
|
||||
await db.insert(serverMetricsRollup).values([
|
||||
{ serverId: old, metric: "cpu", grain: "h", at: ago(metrics.HOURLY_KEEP_DAYS + 3).toISOString(), sum: 1, n: 1, min: 1, max: 1 },
|
||||
{ serverId: old, metric: "cpu", grain: "d", at: ago(metrics.HOURLY_KEEP_DAYS + 3).toISOString(), sum: 1, n: 1, min: 1, max: 1 },
|
||||
{ serverId: old, metric: "cpu", grain: "d", at: ago(metrics.DAILY_KEEP_DAYS + 3).toISOString(), sum: 1, n: 1, min: 1, max: 1 },
|
||||
]);
|
||||
await metrics.recordServerMetrics(old, system(5, 5), NOW);
|
||||
const left = await db.select().from(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, old), eq(serverMetricsRollup.metric, "cpu")));
|
||||
assert.deepEqual(left.map((r) => r.grain).sort(), ["d", "d", "h"], "the old hour is gone, the old day within two years stays, the one beyond is gone, and today's two are there");
|
||||
assert.equal(left.filter((r) => r.grain === "h").length, 1);
|
||||
});
|
||||
|
||||
test("the first time, the hours and days are built from the samples already kept — once, and exactly", async () => {
|
||||
const legacy = (await db.insert(servers).values({ name: "legacy", apiTokenHash: "l", apiTokenPrefix: "l" }).returning({ id: servers.id }))[0].id;
|
||||
const base = new Date("2026-03-10T10:05:00.000Z").getTime();
|
||||
await db.insert(serverMetrics).values([
|
||||
{ serverId: legacy, metric: "cpu", value: 10, recordedAt: new Date(base).toISOString() },
|
||||
{ serverId: legacy, metric: "cpu", value: 20, recordedAt: new Date(base + 10 * 60_000).toISOString() },
|
||||
{ serverId: legacy, metric: "cpu", value: 60, recordedAt: new Date(base + HOUR).toISOString() },
|
||||
]);
|
||||
assert.equal(await metrics.backfillMetricRollups(), true);
|
||||
assert.equal(await metrics.backfillMetricRollups(), false, "done once");
|
||||
const rows = await db.select().from(serverMetricsRollup).where(eq(serverMetricsRollup.serverId, legacy));
|
||||
const hours = rows.filter((r) => r.grain === "h").sort((a, b) => a.at.localeCompare(b.at));
|
||||
assert.deepEqual(hours.map((r) => [r.at, r.n, r.sum, r.min, r.max]), [["2026-03-10T10:00:00.000Z", 2, 30, 10, 20], ["2026-03-10T11:00:00.000Z", 1, 60, 60, 60]]);
|
||||
const days = rows.filter((r) => r.grain === "d");
|
||||
assert.deepEqual(days.map((r) => [r.at, r.n, r.sum, r.min, r.max]), [["2026-03-10T00:00:00.000Z", 3, 90, 10, 60]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("how a check has done over a long time", () => {
|
||||
const probe = (ok: boolean, ms: number | null) => async () => ({ ok, ms, message: ok ? "open" : "refused" });
|
||||
const row = () => db.select().from(endpointChecks).where(eq(endpointChecks.id, checkId)).then((r) => r[0]);
|
||||
|
||||
test("each answer goes into its hour, and the average, the slowest and how many answered are kept", async () => {
|
||||
const t = ago(5, -45 * 60_000); // five days ago, a little into an hour
|
||||
await checks.runCheck(await row(), t, probe(true, 100));
|
||||
await checks.runCheck(await row(), new Date(t.getTime() + 60_000), probe(true, 300));
|
||||
await checks.runCheck(await row(), new Date(t.getTime() + 120_000), probe(false, null));
|
||||
const hour = (await db.select().from(endpointHourly).where(eq(endpointHourly.checkId, checkId)))[0];
|
||||
assert.deepEqual([hour.total, hour.ok, hour.msSum, hour.msCount, hour.msMax], [3, 2, 400, 2, 300]);
|
||||
});
|
||||
|
||||
test("bucketing adds the figures up, so a slice is as true as the checks in it", () => {
|
||||
const rows = [
|
||||
{ t: 0, total: 10, ok: 10, msSum: 1000, msCount: 10, msMax: 400 },
|
||||
{ t: 1000, total: 10, ok: 5, msSum: 1000, msCount: 5, msMax: 900 },
|
||||
{ t: 9000, total: 4, ok: 4, msSum: 400, msCount: 4, msMax: 100 },
|
||||
];
|
||||
const points = checks.bucketHistory(rows, 0, 10_000, 2);
|
||||
assert.deepEqual(points.map((p) => [p.availability, p.avgMs, p.maxMs, p.checks]), [[75, 133, 900, 20], [100, 100, 100, 4]]);
|
||||
assert.deepEqual(checks.bucketHistory([], 0, 10, 2), []);
|
||||
assert.equal(checks.bucketHistory([{ t: 0, total: 1, ok: 0, msSum: 0, msCount: 0, msMax: 0 }], 0, 10, 1)[0].avgMs, null, "no timed answer, no average");
|
||||
});
|
||||
|
||||
test("a day is drawn from the results, a month from the hours, and half a year from the days", async () => {
|
||||
// some results of today, kept in full
|
||||
await db.insert(endpointResults).values([
|
||||
{ checkId, at: ago(0, -30 * 60_000).toISOString(), ok: true, ms: 80, message: "open" },
|
||||
{ checkId, at: ago(0, -20 * 60_000).toISOString(), ok: false, ms: null, message: "refused" },
|
||||
]);
|
||||
// and daily figures for the last 200 days
|
||||
for (let d = 200; d >= 7; d--) {
|
||||
const day = ago(d).toISOString().slice(0, 10);
|
||||
await db.insert(endpointDaily).values({ checkId, day, total: 288, ok: d % 50 === 0 ? 250 : 288, msSum: 288 * 40, msCount: 288 }).onConflictDoNothing();
|
||||
}
|
||||
const day = await checks.getEndpointHistory(checkId, 1, NOW);
|
||||
assert.equal(day.source, "results");
|
||||
assert.deepEqual([day.checks, day.availability, day.avgMs], [2, 50, 80]);
|
||||
|
||||
const month = await checks.getEndpointHistory(checkId, 30, NOW);
|
||||
assert.equal(month.source, "hourly");
|
||||
assert.equal(month.checks, 3, "the three asked five days ago");
|
||||
assert.equal(month.availability, 66.7);
|
||||
assert.equal(month.points[0].maxMs, 300);
|
||||
|
||||
const half = await checks.getEndpointHistory(checkId, 200, NOW);
|
||||
assert.equal(half.source, "daily");
|
||||
assert.ok(half.points.length >= 150 && half.points.length <= 200, "a point a day");
|
||||
assert.ok(half.availability !== null && half.availability < 100 && half.availability > 98);
|
||||
assert.equal(half.avgMs, 40);
|
||||
});
|
||||
|
||||
test("the first time, the hours are built from the results still kept — once", async () => {
|
||||
const fresh = (await db.insert(endpointChecks).values({ name: "Other", kind: "tcp", target: "127.0.0.1", port: 2 }).returning({ id: endpointChecks.id }))[0].id;
|
||||
await db.insert(endpointResults).values([
|
||||
{ checkId: fresh, at: "2026-03-10T10:05:00.000Z", ok: true, ms: 50, message: "" },
|
||||
{ checkId: fresh, at: "2026-03-10T10:35:00.000Z", ok: true, ms: 150, message: "" },
|
||||
{ checkId: fresh, at: "2026-03-10T10:45:00.000Z", ok: false, ms: null, message: "" },
|
||||
]);
|
||||
assert.equal(await checks.backfillEndpointHourly(), true);
|
||||
assert.equal(await checks.backfillEndpointHourly(), false);
|
||||
const [hour] = await db.select().from(endpointHourly).where(eq(endpointHourly.checkId, fresh));
|
||||
assert.deepEqual([hour.hour, hour.total, hour.ok, hour.msSum, hour.msCount, hour.msMax], ["2026-03-10T10:00:00.000Z", 3, 2, 200, 2, 150]);
|
||||
});
|
||||
|
||||
test("the hours are pruned after four months", async () => {
|
||||
await db.insert(endpointHourly).values({ checkId, hour: ago(checks.KEEP_HOURLY_DAYS + 5).toISOString(), total: 1, ok: 1, msSum: 1, msCount: 1, msMax: 1 });
|
||||
await checks.pruneResults(NOW);
|
||||
const left = await db.select().from(endpointHourly).where(eq(endpointHourly.checkId, checkId));
|
||||
assert.ok(left.every((r) => Date.parse(r.hour) > NOW.getTime() - checks.KEEP_HOURLY_DAYS * DAY));
|
||||
});
|
||||
});
|
||||
|
||||
describe("over the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
before(async () => {
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "ada" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/servers", serversRouter);
|
||||
a.use("/api/endpoints", endpointsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
test("a server's metrics can be asked for a year, and no more", async () => {
|
||||
const year = (await (await fetch(`${base}/servers/${serverId}/metrics?days=365`)).json()) as any;
|
||||
assert.equal(year.days, 365);
|
||||
assert.ok(year.series.length > 0);
|
||||
const more = (await (await fetch(`${base}/servers/${serverId}/metrics?days=5000`)).json()) as any;
|
||||
assert.equal(more.days, 365);
|
||||
const none = (await (await fetch(`${base}/servers/${serverId}/metrics`)).json()) as any;
|
||||
assert.equal(none.days, 7);
|
||||
});
|
||||
|
||||
test("a check's history: availability and answer times for a range, 404 for one that is not there", async () => {
|
||||
const body = (await (await fetch(`${base}/endpoints/${checkId}/history?days=30`)).json()) as any;
|
||||
assert.deepEqual([body.check.name, body.days, body.source, body.checks], ["Site", 30, "hourly", 5]); // the three asked five days ago, and the two results of today that the backfill above counted
|
||||
assert.equal((await fetch(`${base}/endpoints/999999/history`)).status, 404);
|
||||
assert.equal((await fetch(`${base}/endpoints/abc/history`)).status, 400);
|
||||
assert.equal(((await (await fetch(`${base}/endpoints/${checkId}/history?days=99999`)).json()) as any).days, 365);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,279 @@
|
||||
/**
|
||||
* The read API and personal tokens: who may make a token, that the secret is shown once and only its hash kept, what a token can read (and that it can only
|
||||
* read), and when it stops working.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-readapi-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
process.env.APP_BASE_URL = "https://hm.example.lan";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, assets, apiTokens, endpointChecks, endpointResults, alertIncidents, maintenanceWindows, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { createApp } = await import("../src/app.js");
|
||||
const { apiTokensRouter } = await import("../src/routes/apiTokens.js");
|
||||
const { setAck } = await import("../src/services/alertAcks.js");
|
||||
const { invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { startOrExtendWindow } = await import("../src/services/maintenance.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const HOUR = 3600_000;
|
||||
let app: Server;
|
||||
let tokensApp: Server;
|
||||
let base = "";
|
||||
let tokensBase = "";
|
||||
let who = "ada";
|
||||
const ids: Record<string, number> = {};
|
||||
|
||||
const api = (path: string, secret: string | null, init: RequestInit = {}) =>
|
||||
fetch(`${base}/api/v1${path}`, { ...init, headers: { ...(secret ? { authorization: `Bearer ${secret}` } : {}), ...(init.headers ?? {}) } });
|
||||
const mine = (path: string, init: RequestInit = {}) => fetch(`${tokensBase}${path}`, { ...init, headers: { "content-type": "application/json", ...(init.headers ?? {}) } });
|
||||
const make = async (label: string, extra: Record<string, unknown> = {}) => {
|
||||
const res = await mine("", { method: "POST", body: JSON.stringify({ label, ...extra }) });
|
||||
return { status: res.status, body: (await res.json()) as any };
|
||||
};
|
||||
|
||||
before(async () => {
|
||||
const rows = await db
|
||||
.insert(users)
|
||||
.values([
|
||||
{ oidcSub: "ada", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "vic", name: "Vic", email: "vic@example.lan", role: "viewer" },
|
||||
])
|
||||
.returning({ id: users.id, oidcSub: users.oidcSub });
|
||||
for (const r of rows) ids[r.oidcSub] = r.id;
|
||||
|
||||
// two servers (one reporting, one gone quiet), an alert on the quiet one, a piece of hardware, a check, an incident and a maintenance window
|
||||
ids.web = (await db.insert(servers).values({ name: "web", apiTokenHash: "SECRET-HASH-WEB", apiTokenPrefix: "w", tags: JSON.stringify(["lab"]), ipAddresses: JSON.stringify(["10.0.0.5"]), lastSeenAt: new Date().toISOString(), cpuLoadPercent: 12, memTotalBytes: 1000, memUsedBytes: 400, disks: JSON.stringify([{ mount: "/", sizeBytes: 100, usedBytes: 40 }]) }).returning({ id: servers.id }))[0].id;
|
||||
ids.db = (await db.insert(servers).values({ name: "db", apiTokenHash: "SECRET-HASH-DB", apiTokenPrefix: "d", lastSeenAt: new Date(Date.now() - 6 * HOUR).toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
await db.insert(assets).values({ name: "Rack switch", kind: "network", location: "Rack A", purchasePrice: 120, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() });
|
||||
ids.check = (await db.insert(endpointChecks).values({ name: "Site", kind: "tcp", target: "127.0.0.1", port: 1, lastOk: true, lastCheckedAt: new Date().toISOString(), lastResponseMs: 12 }).returning({ id: endpointChecks.id }))[0].id;
|
||||
await db.insert(endpointResults).values({ checkId: ids.check, at: new Date().toISOString(), ok: true, ms: 12, message: "open" });
|
||||
await db.insert(alertIncidents).values({ alertKey: "offline:offline:server:1", severity: "critical", peakSeverity: "critical", category: "offline", source: "Server", message: "old", openedAt: new Date(Date.now() - 9 * HOUR).toISOString(), resolvedAt: new Date(Date.now() - 8 * HOUR).toISOString(), lastSeenAt: new Date().toISOString() });
|
||||
await startOrExtendWindow("server", ids.web, 60, "Patching", "Ada");
|
||||
await setAck({ alertId: `offline:offline:server:${ids.db}`, note: "On it", assigneeId: ids.ada, hours: 4, actor: "Ada" });
|
||||
invalidateAlertsCache();
|
||||
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = createApp().listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/", apiTokensRouter);
|
||||
tokensApp = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
tokensBase = `http://127.0.0.1:${(tokensApp.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
tokensApp.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows may still hold the database file
|
||||
}
|
||||
});
|
||||
|
||||
describe("making and keeping tokens", () => {
|
||||
let secret = "";
|
||||
let tokenId = 0;
|
||||
|
||||
test("anyone signed in makes their own; the secret is in the answer once, and only a hash is kept", async () => {
|
||||
who = "vic";
|
||||
const made = await make("Home Assistant", { expiresInDays: 90 });
|
||||
assert.equal(made.status, 201);
|
||||
secret = made.body.secret;
|
||||
tokenId = made.body.token.id;
|
||||
assert.match(secret, /^hlm_p_[A-Za-z0-9_-]{20,}$/);
|
||||
assert.ok(made.body.token.expiresAt, "it ends");
|
||||
const stored = (await db.select().from(apiTokens).where(eq(apiTokens.id, tokenId)))[0];
|
||||
assert.notEqual(stored.tokenHash, secret);
|
||||
assert.equal(stored.tokenHash.length, 64);
|
||||
assert.equal(stored.userId, ids.vic);
|
||||
const list = await (await mine("")).json();
|
||||
assert.deepEqual(list.tokens.map((t: any) => t.label), ["Home Assistant"]);
|
||||
assert.ok(!JSON.stringify(list).includes(secret) && !JSON.stringify(list).includes(stored.tokenHash), "the list never has the secret or the hash");
|
||||
});
|
||||
|
||||
test("a token needs a name, and the end is 30, 90 or 365 days or none", async () => {
|
||||
assert.equal((await make("")).status, 400);
|
||||
assert.equal((await make("x", { expiresInDays: 7 })).status, 400);
|
||||
const forever = await make("No end");
|
||||
assert.equal(forever.status, 201);
|
||||
assert.equal(forever.body.token.expiresAt, null);
|
||||
await mine(`/${forever.body.token.id}`, { method: "DELETE" });
|
||||
});
|
||||
|
||||
test("a person has at most ten", async () => {
|
||||
who = "ada";
|
||||
for (let i = 0; i < 10; i++) assert.equal((await make(`t${i}`)).status, 201);
|
||||
const eleventh = await make("one too many");
|
||||
assert.equal(eleventh.status, 400);
|
||||
assert.match(eleventh.body.message, /already 10/);
|
||||
for (const t of (await (await mine("")).json()).tokens) await mine(`/${t.id}`, { method: "DELETE" });
|
||||
});
|
||||
|
||||
test("somebody else's token is not yours to see or remove", async () => {
|
||||
who = "ada";
|
||||
assert.equal(((await (await mine("")).json()) as any).tokens.length, 0, "Ada sees none of Vic's");
|
||||
assert.equal((await mine(`/${tokenId}`, { method: "DELETE" })).status, 404);
|
||||
assert.equal((await api("/me", secret)).status, 200, "and it still works");
|
||||
});
|
||||
|
||||
test("removing your own stops it at once", async () => {
|
||||
who = "vic";
|
||||
assert.equal((await api("/me", secret)).status, 200);
|
||||
assert.equal((await mine(`/${tokenId}`, { method: "DELETE" })).status, 204);
|
||||
assert.equal((await api("/me", secret)).status, 401);
|
||||
const entries = (await db.select().from(auditLog)).filter((e) => e.category === "api_token");
|
||||
assert.ok(entries.some((e) => e.action === "create") && entries.some((e) => e.action === "delete"), "making and removing a token is written down");
|
||||
assert.ok(entries.every((e) => !JSON.stringify(e).includes("hlm_p_")), "and the secret is not");
|
||||
});
|
||||
});
|
||||
|
||||
describe("what a token can read", () => {
|
||||
let secret = "";
|
||||
|
||||
before(async () => {
|
||||
who = "vic";
|
||||
secret = (await make("Reader")).body.secret;
|
||||
});
|
||||
|
||||
test("no token, a wrong one, and one in the address are all the same 401", async () => {
|
||||
for (const res of [await api("/servers", null), await api("/servers", "hlm_p_wrongwrongwrongwrongwrong"), await fetch(`${base}/api/v1/servers?token=${secret}`)]) {
|
||||
assert.equal(res.status, 401);
|
||||
assert.equal(res.headers.get("www-authenticate"), 'Bearer realm="homelab-manager"');
|
||||
assert.equal(((await res.json()) as any).error, "unauthorized");
|
||||
}
|
||||
});
|
||||
|
||||
test("it only reads: any other verb is refused, even with a good token", async () => {
|
||||
for (const method of ["POST", "PUT", "PATCH", "DELETE"]) {
|
||||
const res = await api("/servers", secret, { method });
|
||||
assert.equal(res.status, 405, method);
|
||||
assert.equal(res.headers.get("allow"), "GET, HEAD");
|
||||
}
|
||||
});
|
||||
|
||||
test("who the token is", async () => {
|
||||
const body = (await (await api("/me", secret)).json()) as any;
|
||||
assert.deepEqual([body.user.name, body.user.role, body.token.label], ["Vic", "viewer", "Reader"]);
|
||||
});
|
||||
|
||||
test("servers: a stable shape, no token hashes, the state worked out, filtered by tag and status", async () => {
|
||||
const res = await api("/servers", secret);
|
||||
assert.equal(res.headers.get("cache-control"), "no-store");
|
||||
const text = await res.text();
|
||||
assert.ok(!text.includes("SECRET-HASH") && !text.includes("apiToken"), "nothing of the servers' own tokens");
|
||||
const body = JSON.parse(text) as any;
|
||||
const web = body.servers.find((s: any) => s.name === "web");
|
||||
assert.deepEqual([web.status, web.tags, web.ipAddresses, web.cpu.loadPercent, web.disks[0].usedPercent, web.url], ["online", ["lab"], ["10.0.0.5"], 12, 40, `https://hm.example.lan/servers/${ids.web}`]);
|
||||
assert.equal(body.servers.find((s: any) => s.name === "db").status, "offline");
|
||||
assert.deepEqual(((await (await api("/servers?tag=lab", secret)).json()) as any).servers.map((s: any) => s.name), ["web"]);
|
||||
assert.deepEqual(((await (await api("/servers?status=offline", secret)).json()) as any).servers.map((s: any) => s.name), ["db"]);
|
||||
assert.equal((await api(`/servers/${ids.web}`, secret)).status, 200);
|
||||
assert.equal((await api("/servers/99999", secret)).status, 404);
|
||||
assert.equal((await api("/servers/abc", secret)).status, 400);
|
||||
});
|
||||
|
||||
test("a server's metrics, for up to a year", async () => {
|
||||
const body = (await (await api(`/servers/${ids.web}/metrics?days=9999`, secret)).json()) as any;
|
||||
assert.equal(body.days, 365);
|
||||
assert.ok(Array.isArray(body.series));
|
||||
});
|
||||
|
||||
test("alerts: what is wrong, with whether somebody is on it, filterable; addresses are whole", async () => {
|
||||
const body = (await (await api("/alerts", secret)).json()) as any;
|
||||
const offline = body.alerts.find((a: any) => a.id === `offline:offline:server:${ids.db}`);
|
||||
assert.ok(offline, "the quiet server is raised");
|
||||
assert.deepEqual([offline.severity, offline.acknowledgement.by, offline.acknowledgement.assignee, offline.acknowledgement.note], ["critical", "Ada", "Ada", "On it"]);
|
||||
assert.equal(offline.url, `https://hm.example.lan/servers/${ids.db}`);
|
||||
assert.equal(body.needingAttention.acknowledged, 1);
|
||||
assert.equal(((await (await api("/alerts?acknowledged=false", secret)).json()) as any).alerts.some((a: any) => a.id === offline.id), false);
|
||||
assert.equal(((await (await api("/alerts?severity=info", secret)).json()) as any).alerts.length, 0);
|
||||
});
|
||||
|
||||
test("incidents, inventory, network, endpoints and maintenance", async () => {
|
||||
const incidents = (await (await api("/incidents?status=resolved", secret)).json()) as any;
|
||||
assert.equal(incidents.incidents.length, 1);
|
||||
assert.equal(((await (await api(`/incidents/${incidents.incidents[0].id}`, secret)).json()) as any).incident?.id ?? incidents.incidents[0].id, incidents.incidents[0].id);
|
||||
const open = ((await (await api("/incidents?status=open", secret)).json()) as any).incidents;
|
||||
assert.deepEqual(open.map((i: any) => i.alertKey), [`offline:offline:server:${ids.db}`], "the quiet server's alert is an incident that is still open");
|
||||
|
||||
const inventory = (await (await api("/inventory", secret)).json()) as any;
|
||||
assert.deepEqual([inventory.assets[0].name, inventory.assets[0].location, inventory.assets[0].purchasePrice], ["Rack switch", "Rack A", 120]);
|
||||
|
||||
const network = (await (await api("/network", secret)).json()) as any;
|
||||
assert.deepEqual(Object.keys(network).sort(), ["devices", "networks", "serverLinks", "ssids"]);
|
||||
|
||||
const endpoints = (await (await api("/endpoints", secret)).json()) as any;
|
||||
assert.deepEqual([endpoints.endpoints[0].name, endpoints.endpoints[0].status, endpoints.endpoints[0].lastResponseMs], ["Site", "up", 12]);
|
||||
const history = (await (await api(`/endpoints/${ids.check}/history?days=1`, secret)).json()) as any;
|
||||
assert.deepEqual([history.check.name, history.source, history.checks], ["Site", "results", 1]);
|
||||
assert.equal((await api("/endpoints/99999/history", secret)).status, 404);
|
||||
|
||||
const maintenance = (await (await api("/maintenance", secret)).json()) as any;
|
||||
assert.deepEqual([maintenance.active.length, maintenance.active[0].target.name, maintenance.active[0].reason], [1, "web", "Patching"]);
|
||||
});
|
||||
|
||||
test("an address that is not there says so, and the index lists them", async () => {
|
||||
const res = await api("/nothing", secret);
|
||||
assert.equal(res.status, 404);
|
||||
const index = (await (await api("/", secret)).json()) as any;
|
||||
assert.ok(index.endpoints.some((e: string) => e.includes("/api/v1/alerts")));
|
||||
});
|
||||
});
|
||||
|
||||
describe("when a token stops working", () => {
|
||||
test("it ends on the day it was made to; a removed person's tokens go with them", async () => {
|
||||
who = "ada";
|
||||
const { body } = await make("Short");
|
||||
const secret = body.secret;
|
||||
assert.equal((await api("/me", secret)).status, 200);
|
||||
// the end comes: move it into the past
|
||||
await db.update(apiTokens).set({ expiresAt: new Date(Date.now() - 1000).toISOString() }).where(eq(apiTokens.id, body.token.id));
|
||||
assert.equal((await api("/me", secret)).status, 401, "expired");
|
||||
const listed = ((await (await mine("")).json()) as any).tokens.find((t: any) => t.id === body.token.id);
|
||||
assert.equal(listed.expired, true, "and the list says so");
|
||||
|
||||
const gone = (await db.insert(users).values({ oidcSub: "temp", name: "Temp", role: "viewer" }).returning({ id: users.id }))[0].id;
|
||||
const { createApiToken } = await import("../src/services/apiTokens.js");
|
||||
const t = await createApiToken({ userId: gone, label: "Leaver", expiresInDays: null });
|
||||
assert.equal((await api("/me", t.token)).status, 200);
|
||||
await db.delete(users).where(eq(users.id, gone));
|
||||
assert.equal((await api("/me", t.token)).status, 401, "the person is gone, so is the token");
|
||||
assert.equal((await db.select().from(apiTokens).where(eq(apiTokens.userId, gone))).length, 0);
|
||||
});
|
||||
|
||||
test("use is noted, but not more than once a minute", async () => {
|
||||
who = "ada";
|
||||
const { body } = await make("Used");
|
||||
assert.equal((await db.select().from(apiTokens).where(eq(apiTokens.id, body.token.id)))[0].lastUsedAt, null);
|
||||
await api("/me", body.secret);
|
||||
const first = (await db.select().from(apiTokens).where(eq(apiTokens.id, body.token.id)))[0].lastUsedAt;
|
||||
assert.ok(first);
|
||||
await api("/me", body.secret);
|
||||
assert.equal((await db.select().from(apiTokens).where(eq(apiTokens.id, body.token.id)))[0].lastUsedAt, first);
|
||||
});
|
||||
});
|
||||
@@ -23,7 +23,7 @@ const { users, servers, serverMetrics } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { serversRouter } = await import("../src/routes/servers.js");
|
||||
const { syncServerTasks } = await import("../src/services/taskSync.js");
|
||||
const { samplesFromSystem, recordServerMetrics, bucketPoints, trendOf, getServerMetrics, MIN_GAP_MS, RETENTION_DAYS } = await import("../src/services/serverMetrics.js");
|
||||
const { samplesFromSystem, recordServerMetrics, bucketPoints, trendOf, getServerMetrics, MIN_GAP_MS, RETENTION_DAYS, MAX_RANGE_DAYS } = await import("../src/services/serverMetrics.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
@@ -189,8 +189,8 @@ describe("GET /api/servers/:id/metrics", () => {
|
||||
test("anyone signed in can read it, and the number of days is kept within what is stored", async () => {
|
||||
const res = await fetch(`${base}/${serverId}/metrics?days=9999`);
|
||||
assert.equal(res.status, 200);
|
||||
const body = await res.json();
|
||||
assert.equal(body.days, RETENTION_DAYS);
|
||||
assert.equal((await res.json()).days, MAX_RANGE_DAYS, "a year at most: the rest is kept as hours and days");
|
||||
const body = await (await fetch(`${base}/${serverId}/metrics?days=${RETENTION_DAYS}`)).json();
|
||||
assert.deepEqual(body.series.map((s: { kind: string }) => s.kind), ["cpu", "mem", "disk"]);
|
||||
assert.equal((await fetch(`${base}/${serverId}/metrics`)).status, 200);
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user