A year of history, one chart, and a read API with personal tokens

History: server load, memory and disks are also kept as hourly (four months) and
daily (over two years) averages with lowest and highest; endpoint answers as
hourly (four months) beside the daily figures. Charts ask for 24 hours to a year
and are drawn from whichever is fine enough. A disk that two weeks call flat is
looked at again over ninety days for the forecast. Built once from what is kept.

One chart: TimeChart replaces the sparkline and the multi-line chart: hover and
arrow-key readout, a lowest/highest band, gaps shown as gaps, the same ranges
everywhere and the choice remembered. Used for server trends, comparing servers,
a check's availability and answer times (Endpoints > History) and a server's
availability.

Read API: /api/v1 (servers, metrics, alerts, incidents, inventory, network,
endpoints and their history, maintenance) with personal read-only tokens made
under the user menu: shown once, hashed, 30/90/365 days or none, ten at most,
removed with their owner. GET only. Documented in API.md.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-20 20:22:58 +02:00
1 parent 5e26e77340
commit 68b41b5586
47 files changed
+12360 -247

No files matched your search

+5
View File
@@ -47,6 +47,8 @@ import { alertsRouter } from "./routes/alerts.js";
import { mapRouter } from "./routes/map.js";
import { networkRouter } from "./routes/network.js";
import { myWorkRouter } from "./routes/myWork.js";
import { apiTokensRouter } from "./routes/apiTokens.js";
import { publicApiRouter } from "./routes/publicApi.js";
import { generatorRouter } from "./routes/generator.js";
import { alertTicketsRouter } from "./routes/alertTickets.js";
import { avatarsRouter } from "./routes/avatars.js";
@@ -83,6 +85,8 @@ export function createApp() {
app.use("/api/agent/report", agentReportRouter);
// A job reporting that it ran: a token in the address, no session, and no browser to be checked for where it came from.
app.use("/api/heartbeat", heartbeatPingRouter);
// The read API: a personal token in a header, GET only, no session and nothing for the same-origin check to protect.
app.use("/api/v1", publicApiRouter);
// Everything after this is reached with a session cookie, so a request that came from another origin is refused here.
app.use(sameOriginGuard);
app.use(express.json());
@@ -114,6 +118,7 @@ export function createApp() {
}
app.use("/auth", authRouter);
app.use("/api/me/tokens", apiTokensRouter);
app.use("/api/me", meRouter);
app.use("/api/avatars", avatarsRouter);
app.use("/api/heartbeats", heartbeatsRouter);
+57
View File
@@ -263,6 +263,26 @@ export const serverMetrics = sqliteTable(
(table) => [index("server_metrics_lookup_idx").on(table.serverId, table.metric, table.recordedAt)],
);
// The same figures, kept longer and coarser: one row per server, figure and hour (grain "h", kept a few months) or day (grain "d", kept for years), with what is
// needed to draw them — the sum and count (so the average can be worked out as more samples arrive), and the lowest and highest. `at` is the start of the hour
// or day, as ISO. Written as each sample is recorded; see services/serverMetrics.ts.
export const serverMetricsRollup = sqliteTable(
"server_metrics_rollup",
{
serverId: integer("server_id")
.notNull()
.references(() => servers.id, { onDelete: "cascade" }),
metric: text("metric").notNull(),
grain: text("grain").$type<"h" | "d">().notNull(),
at: text("at").notNull(),
sum: real("sum").notNull(),
n: integer("n").notNull(),
min: real("min").notNull(),
max: real("max").notNull(),
},
(t) => [primaryKey({ columns: [t.serverId, t.metric, t.grain, t.at] })],
);
// ─── Settings (key/value) ───────────────────────────────────────────────────
export const settings = sqliteTable("settings", {
@@ -993,6 +1013,24 @@ export const endpointDaily = sqliteTable(
(t) => [primaryKey({ columns: [t.checkId, t.day] })],
);
// The same, by the hour: how many times a check was asked in that hour, how many answered, and the answer times — kept a few months, between the individual
// results (two weeks) and the daily figures (years), so an answer-time chart can reach back beyond two weeks without a point for every check.
export const endpointHourly = sqliteTable(
"endpoint_hourly",
{
checkId: integer("check_id")
.notNull()
.references(() => endpointChecks.id, { onDelete: "cascade" }),
hour: text("hour").notNull(), // ISO, the start of the hour (UTC)
total: integer("total").notNull().default(0),
ok: integer("ok").notNull().default(0),
msSum: integer("ms_sum").notNull().default(0),
msCount: integer("ms_count").notNull().default(0),
msMax: integer("ms_max").notNull().default(0),
},
(t) => [primaryKey({ columns: [t.checkId, t.hour] })],
);
// A certificate, domain registration or secret whose expiry moved later: it was renewed. Written when the app sees it happen, so it only goes back
// as far as the version that began watching (services/renewals.ts).
export const renewals = sqliteTable(
@@ -1050,6 +1088,25 @@ export const metricsTokens = sqliteTable("metrics_tokens", {
lastUsedAt: text("last_used_at"), // ISO
});
// ─── Personal API tokens: read-only access to /api/v1 for scripts and Home Assistant ─────
// One person's own: made by them, shown to them once, and gone when they are. Only a hash of the secret is kept. A token only ever reads (the API has no other verbs),
// and sees what any signed-in person sees.
export const apiTokens = sqliteTable("api_tokens", {
id: integer("id").primaryKey({ autoIncrement: true }),
userId: integer("user_id")
.notNull()
.references(() => users.id, { onDelete: "cascade" }),
label: text("label").notNull(), // e.g. "Home Assistant"
tokenHash: text("token_hash").notNull().unique(),
tokenPrefix: text("token_prefix").notNull(),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
lastUsedAt: text("last_used_at"), // ISO
expiresAt: text("expires_at"), // ISO; null = until removed
});
// ─── Event webhooks: tell another system when something happens ─────────────
export const webhookEvents = ["alert.opened", "alert.resolved", "maintenance.started", "maintenance.ended"] as const;
+4
View File
@@ -25,6 +25,8 @@ import { initEndpointMonitor } from "./services/endpointMonitor.js";
import { initPatchPlans } from "./services/patchPlans.js";
import { initFlapMonitor } from "./services/flapMonitor.js";
import { initUnifiSync } from "./services/unifiSync.js";
import { backfillMetricRollups } from "./services/serverMetrics.js";
import { backfillEndpointHourly } from "./services/endpointChecks.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
import { initIncidentRecorder } from "./services/incidents.js";
import { getAlerts } from "./services/alerts.js";
@@ -69,6 +71,8 @@ initEndpointMonitor();
initPatchPlans();
initFlapMonitor();
initUnifiSync();
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
createApp().listen(env.port, () => {
console.log(`homelab-manager listening on port ${env.port}`);
+46
View File
@@ -0,0 +1,46 @@
import { Router } from "express";
import { z } from "zod";
import { requireAuth } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { createApiToken, EXPIRY_CHOICES, listApiTokens, MAX_TOKENS_PER_PERSON, publicApiToken, removeApiToken, TokenLimitError } from "../services/apiTokens.js";
import { asyncHandler } from "../utils/asyncHandler.js";
/** A person's own API tokens: anyone signed in makes, sees and removes theirs — never anybody else's. */
export const apiTokensRouter = Router();
apiTokensRouter.use(requireAuth);
const createSchema = z.object({
label: z.string().trim().min(1).max(60),
// Days until it stops working; null (or left out) for until it is removed.
expiresInDays: z.union([z.literal(30), z.literal(90), z.literal(365)]).nullable().optional(),
});
apiTokensRouter.get("/", asyncHandler(async (req, res) => {
const rows = await listApiTokens(req.currentUser!.id);
res.json({ tokens: rows.map((r) => publicApiToken(r)), max: MAX_TOKENS_PER_PERSON, expiryChoices: EXPIRY_CHOICES });
}));
// The secret is in this answer and nowhere else, ever.
apiTokensRouter.post("/", asyncHandler(async (req, res) => {
const parsed = createSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give the token a name, like Home Assistant, and choose 30, 90 or 365 days or no end." });
const actor = req.currentUser!;
try {
const { row, token } = await createApiToken({ userId: actor.id, label: parsed.data.label, expiresInDays: parsed.data.expiresInDays ?? null });
await recordAudit({ actor, category: "api_token", action: "create", targetType: "api_token", targetId: row.id, detail: { label: row.label, expiresAt: row.expiresAt } });
res.status(201).json({ token: publicApiToken(row), secret: token });
} catch (err) {
if (err instanceof TokenLimitError) return res.status(400).json({ error: "too_many", message: err.message });
throw err;
}
}));
apiTokensRouter.delete("/:id", asyncHandler(async (req, res) => {
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const actor = req.currentUser!;
const removed = await removeApiToken(actor.id, id);
if (!removed) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor, category: "api_token", action: "delete", targetType: "api_token", targetId: id, detail: { label: removed.label } });
res.status(204).end();
}));
+12 -1
View File
@@ -6,7 +6,7 @@ import { endpointChecks, servers } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { CHECK_KINDS, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
import { CHECK_KINDS, getEndpointHistory, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
import { loadDismissed, loadSuggestions, saveDismissed } from "../services/endpointSuggestions.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -63,6 +63,17 @@ endpointsRouter.get("/", asyncHandler(async (_req, res) => {
});
}));
// How one check has done over a day, a week, a month, three months or a year: availability and answer times, for the chart on its page.
endpointsRouter.get("/:id/history", asyncHandler(async (req, res) => {
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [check] = await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
if (!check) return res.status(404).json({ error: "not_found" });
const requested = Number(req.query.days);
const days = Number.isFinite(requested) ? Math.min(365, Math.max(1, Math.round(requested))) : 7;
res.json({ check, ...(await getEndpointHistory(id, days)) });
}));
function valuesOf(d: z.infer<typeof checkSchema>) {
return {
name: d.name,
+4 -63
View File
@@ -2,12 +2,11 @@ import { Router } from "express";
import { eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { integrations, networkDevices, networkSegments, networkServerLinks, servers } from "../db/schema.js";
import { integrations } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { dependentsOf, loadGraph, needsOfNode, nodeKey } from "../services/dependencies.js";
import { activeSubjects } from "../services/maintenance.js";
import { syncAllUnifi, syncStatuses, syncUnifiIntegration } from "../services/unifiSync.js";
import { buildNetworkOverview } from "../services/networkOverview.js";
import { syncAllUnifi, syncUnifiIntegration } from "../services/unifiSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const networkRouter = Router();
@@ -19,65 +18,7 @@ networkRouter.use(requireAuth);
* names. Everyone signed in can see it (it is the same picture the dependency map and the alerts draw on); only starting a fresh read needs more.
*/
networkRouter.get("/", asyncHandler(async (_req, res) => {
const [intRows, devices, segments, statuses, graph, silenced, linkRows, serverRows] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "unifi")),
db.select().from(networkDevices),
db.select().from(networkSegments),
syncStatuses(),
loadGraph(),
activeSubjects(),
db.select().from(networkServerLinks),
db.select({ id: servers.id, name: servers.name }).from(servers),
]);
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
const intName = new Map(intRows.map((i) => [i.id, i.name]));
const byExternal = new Map(devices.map((d) => [`${d.integrationId}:${d.externalId}`, d]));
// The servers plugged into each device, by what the controller says is connected (see unifiSync.matchClientsToServers).
const plugged = new Map<number, { serverId: number; serverName: string; port: number | null; wired: boolean }[]>();
const serverLinks: { serverId: number; serverName: string; deviceId: number; deviceName: string; port: number | null; wired: boolean }[] = [];
for (const l of linkRows) {
const device = byExternal.get(`${l.integrationId}:${l.deviceExternalId}`);
const name = serverName.get(l.serverId);
if (!device || !name) continue;
plugged.set(device.id, [...(plugged.get(device.id) ?? []), { serverId: l.serverId, serverName: name, port: l.port, wired: l.wired }]);
serverLinks.push({ serverId: l.serverId, serverName: name, deviceId: device.id, deviceName: device.name, port: l.port, wired: l.wired });
}
res.json({
serverLinks,
integrations: intRows.map((i) => ({ id: i.id, name: i.name, enabled: i.enabled, status: statuses[i.id] ?? null })),
devices: devices
.map((d) => {
const key = nodeKey("device", d.id);
const uplink = d.uplinkExternalId ? byExternal.get(`${d.integrationId}:${d.uplinkExternalId}`) : undefined;
return {
id: d.id,
integrationId: d.integrationId,
integrationName: intName.get(d.integrationId) ?? "UniFi",
siteName: d.siteName,
name: d.name,
model: d.model,
kind: d.kind,
ip: d.ip,
mac: d.mac,
state: d.state,
uptimeSeconds: d.uptimeSeconds,
firmware: d.firmware,
firmwareUpdatable: d.firmwareUpdatable,
uplink: uplink ? { id: uplink.id, name: uplink.name } : null,
offlineSince: d.offlineSince,
lastOnlineAt: d.lastOnlineAt,
updatedAt: d.updatedAt,
// How many things depend on it (directly or through others), and how many it needs: what the dependency map says about it.
dependents: dependentsOf(graph.edges, key).length,
needs: needsOfNode(graph.edges, key).length,
inMaintenance: silenced.has(key),
servers: (plugged.get(d.id) ?? []).sort((a, b) => a.serverName.localeCompare(b.serverName)),
};
})
.sort((a, b) => a.name.localeCompare(b.name)),
networks: segments.filter((s) => s.kind === "network").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => (a.vlanId ?? 0) - (b.vlanId ?? 0) || a.name.localeCompare(b.name)),
ssids: segments.filter((s) => s.kind === "ssid").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => a.name.localeCompare(b.name)),
});
res.json(await buildNetworkOverview());
}));
const syncSchema = z.object({ integrationId: z.number().int().positive().optional() });
+165
View File
@@ -0,0 +1,165 @@
import { Router, type NextFunction, type Request, type Response } from "express";
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, servers } from "../db/schema.js";
import { resolveApiToken, type ApiTokenRow, type ApiTokenUser } from "../services/apiTokens.js";
import { getEndpointHistory } from "../services/endpointChecks.js";
import { getIncident, listIncidents } from "../services/incidents.js";
import { buildNetworkOverview } from "../services/networkOverview.js";
import { alertsV1, endpointsV1, inventoryV1, maintenanceV1, oneServerV1, serversV1 } from "../services/publicApi.js";
import { getServerMetrics, MAX_RANGE_DAYS } from "../services/serverMetrics.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { rateLimit } from "../utils/rateLimit.js";
/**
* The read API: JSON about servers, alerts, incidents, inventory, network equipment, endpoint checks and maintenance, for scripts and things like Home Assistant.
* A personal token in the Authorization header (never the address, which ends up in logs); GET only; and nothing here writes anything.
*/
export const publicApiRouter = Router();
declare module "express-serve-static-core" {
interface Request {
apiToken?: { token: ApiTokenRow; user: ApiTokenUser };
}
}
const limiter = rateLimit({ windowMs: 60_000, max: 300, message: "Too many requests." });
publicApiRouter.use(limiter);
publicApiRouter.use((req: Request, res: Response, next: NextFunction) => {
res.setHeader("Cache-Control", "no-store");
if (req.method !== "GET" && req.method !== "HEAD") {
res.setHeader("Allow", "GET, HEAD");
return res.status(405).json({ error: "read_only", message: "This API only reads. It has no way to change anything." });
}
next();
});
publicApiRouter.use(
asyncHandler(async (req, res, next) => {
const match = /^Bearer\s+(\S+)$/i.exec(req.get("authorization") ?? "");
const found = match ? await resolveApiToken(match[1]) : null;
if (!found) {
// The same answer for no token, a wrong one, an expired one and a removed one.
res.setHeader("WWW-Authenticate", 'Bearer realm="homelab-manager"');
return res.status(401).json({ error: "unauthorized", message: "A token is needed: send it as 'Authorization: Bearer <token>'. Make one from the user menu → API tokens." });
}
req.apiToken = found;
next();
}),
);
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
const daysOf = (v: unknown, fallback: number) => {
const n = Number(v);
return Number.isFinite(n) ? Math.min(MAX_RANGE_DAYS, Math.max(1, Math.round(n))) : fallback;
};
publicApiRouter.get("/", (_req, res) => {
res.json({
name: "Homelab Manager read API",
version: 1,
endpoints: [
"GET /api/v1/me",
"GET /api/v1/servers?tag=&status=",
"GET /api/v1/servers/:id",
"GET /api/v1/servers/:id/metrics?days=",
"GET /api/v1/alerts?severity=&category=&acknowledged=&silenced=",
"GET /api/v1/incidents?status=&limit=",
"GET /api/v1/incidents/:id",
"GET /api/v1/inventory",
"GET /api/v1/network",
"GET /api/v1/endpoints",
"GET /api/v1/endpoints/:id/history?days=",
"GET /api/v1/maintenance",
],
});
});
publicApiRouter.get("/me", (req, res) => {
const { token, user } = req.apiToken!;
res.json({ user: { id: user.id, name: user.name, email: user.email, role: user.role }, token: { id: token.id, label: token.label, expiresAt: token.expiresAt } });
});
publicApiRouter.get("/servers", asyncHandler(async (req, res) => {
let list = await serversV1();
if (typeof req.query.tag === "string" && req.query.tag) {
const tag = req.query.tag.toLowerCase();
list = list.filter((s) => s.tags.includes(tag));
}
if (req.query.status === "online" || req.query.status === "offline" || req.query.status === "unknown") list = list.filter((s) => s.status === req.query.status);
res.json({ servers: list });
}));
publicApiRouter.get("/servers/:id", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const server = await oneServerV1(id);
if (!server) return res.status(404).json({ error: "not_found" });
res.json({ server });
}));
publicApiRouter.get("/servers/:id/metrics", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [row] = await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1);
if (!row) return res.status(404).json({ error: "not_found" });
const days = daysOf(req.query.days, 7);
res.json({ days, series: await getServerMetrics(id, days) });
}));
publicApiRouter.get("/alerts", asyncHandler(async (req, res) => {
const report = await alertsV1();
let alerts = report.alerts;
const q = req.query;
if (typeof q.severity === "string") alerts = alerts.filter((a) => a.severity === q.severity);
if (typeof q.category === "string") alerts = alerts.filter((a) => a.category === q.category);
if (q.acknowledged === "true" || q.acknowledged === "false") alerts = alerts.filter((a) => (a.acknowledgement !== null) === (q.acknowledged === "true"));
if (q.silenced === "true" || q.silenced === "false") alerts = alerts.filter((a) => a.silenced === (q.silenced === "true"));
res.json({ ...report, alerts });
}));
publicApiRouter.get("/incidents", asyncHandler(async (req, res) => {
const limit = Math.min(500, Math.max(1, Math.round(Number(req.query.limit)) || 100));
let list = await listIncidents({ limit });
if (req.query.status === "open") list = list.filter((i) => i.open);
else if (req.query.status === "resolved") list = list.filter((i) => !i.open);
res.json({ incidents: list });
}));
publicApiRouter.get("/incidents/:id", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const detail = await getIncident(id);
if (!detail) return res.status(404).json({ error: "not_found" });
res.json(detail);
}));
publicApiRouter.get("/inventory", asyncHandler(async (_req, res) => {
res.json({ assets: await inventoryV1() });
}));
publicApiRouter.get("/network", asyncHandler(async (_req, res) => {
const overview = await buildNetworkOverview();
res.json({ devices: overview.devices, networks: overview.networks, ssids: overview.ssids, serverLinks: overview.serverLinks });
}));
publicApiRouter.get("/endpoints", asyncHandler(async (_req, res) => {
res.json({ endpoints: await endpointsV1() });
}));
publicApiRouter.get("/endpoints/:id/history", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [check] = await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
if (!check) return res.status(404).json({ error: "not_found" });
res.json({ check, ...(await getEndpointHistory(id, daysOf(req.query.days, 7))) });
}));
publicApiRouter.get("/maintenance", asyncHandler(async (_req, res) => {
res.json(await maintenanceV1());
}));
publicApiRouter.use((_req, res) => {
res.status(404).json({ error: "not_found", message: "No such address. GET /api/v1 lists them." });
});
+2 -2
View File
@@ -12,7 +12,7 @@ import { markIgnoredServices, parseHostHealth } from "../services/hostHealth.js"
import { describeDiagnostics, parseDiagnostics } from "../services/agentDiagnostics.js";
import { checkThresholds, parseThresholds } from "../services/serverThresholds.js";
import { latestAgentVersions } from "../services/agentVersions.js";
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
import { getServerMetrics, MAX_RANGE_DAYS } from "../services/serverMetrics.js";
import { getServerActivity } from "../services/serverActivity.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { ID_PATTERNS } from "../utils/pathSegment.js";
@@ -272,7 +272,7 @@ serversRouter.get("/:id/metrics", asyncHandler(async (req, res) => {
const [server] = await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1);
if (!server) return res.status(404).json({ error: "not_found" });
const requested = Number(req.query.days);
const days = Number.isFinite(requested) ? Math.min(RETENTION_DAYS, Math.max(1, Math.round(requested))) : 7;
const days = Number.isFinite(requested) ? Math.min(MAX_RANGE_DAYS, Math.max(1, Math.round(requested))) : 7;
res.json({ days, series: await getServerMetrics(id, days) });
}));
+5 -2
View File
@@ -6,6 +6,9 @@ import { getUptime } from "../services/uptimeHistory.js";
export const uptimeRouter = Router();
// Anyone signed in who can see the server and integration lists can see how those have been doing.
uptimeRouter.get("/", requireAuth, asyncHandler(async (_req, res) => {
res.json(await getUptime());
// `?days=` is 30 by default and at most a year (as far back as the audit log it is worked out from goes).
uptimeRouter.get("/", requireAuth, asyncHandler(async (req, res) => {
const requested = Number(req.query.days);
const days = Number.isFinite(requested) ? Math.min(365, Math.max(1, Math.round(requested))) : undefined;
res.json(await getUptime(new Date(), days));
}));
+2 -2
View File
@@ -12,7 +12,7 @@
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, heartbeats, integrations, networkDevices, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
import { recordFromReport } from "./incidents.js";
@@ -499,7 +499,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
const own = parseThresholds(s.thresholds);
const forecastDays = effectiveThresholds(healthChecks, own).diskForecastDays;
if (!(forecastDays > 0)) continue;
const series = await getServerMetrics(s.id, 14);
const series = await getForecastSeries(s.id);
for (const f of disksFillingSoon(series, forecastDays, (mount) => diskLimit(own, mount, healthChecks.diskUsagePercent))) {
push({
category: "disk",
+74
View File
@@ -0,0 +1,74 @@
/**
* Personal API tokens: read-only access to /api/v1 for scripts, Home Assistant and the like. Each belongs to the person who made it — they see and remove their own —
* and goes when they do. Like an agent token or a share link, the secret is shown once, when it is made, and only its SHA-256 is kept.
*/
import { createHash, randomBytes } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { apiTokens, users } from "../db/schema.js";
export type ApiTokenRow = typeof apiTokens.$inferSelect;
export type ApiTokenUser = typeof users.$inferSelect;
const hash = (token: string) => createHash("sha256").update(token).digest("hex");
/** Most tokens one person may have: one for each thing that reads, and a spare. */
export const MAX_TOKENS_PER_PERSON = 10;
/** How long a token may be asked to last, in days (or never). */
export const EXPIRY_CHOICES = [30, 90, 365] as const;
/** Use is not recorded more than once in this time, so a script that reads every few seconds doesn't write the database each time. */
const USED_WRITE_GAP_MS = 60_000;
export class TokenLimitError extends Error {}
export async function createApiToken(params: { userId: number; label: string; expiresInDays: number | null }, now: Date = new Date()): Promise<{ row: ApiTokenRow; token: string }> {
const existing = await db.select({ id: apiTokens.id }).from(apiTokens).where(eq(apiTokens.userId, params.userId));
if (existing.length >= MAX_TOKENS_PER_PERSON) throw new TokenLimitError(`There are already ${MAX_TOKENS_PER_PERSON} tokens. Remove one you don't use first.`);
const token = `hlm_p_${randomBytes(24).toString("base64url")}`;
const expiresAt = params.expiresInDays === null ? null : new Date(now.getTime() + params.expiresInDays * 86_400_000).toISOString();
const [row] = await db.insert(apiTokens).values({ userId: params.userId, label: params.label, tokenHash: hash(token), tokenPrefix: token.slice(0, 12), expiresAt }).returning();
return { row, token };
}
/** The token a secret belongs to and whose it is, if it is one that still works; notes that it was used. */
export async function resolveApiToken(secret: string, now: Date = new Date()): Promise<{ token: ApiTokenRow; user: ApiTokenUser } | null> {
if (typeof secret !== "string" || secret.length < 20 || secret.length > 128) return null;
const [found] = await db
.select({ token: apiTokens, user: users })
.from(apiTokens)
.innerJoin(users, eq(users.id, apiTokens.userId))
.where(eq(apiTokens.tokenHash, hash(secret)))
.limit(1);
if (!found) return null;
if (found.token.expiresAt && Date.parse(found.token.expiresAt) <= now.getTime()) return null;
if (!found.token.lastUsedAt || now.getTime() - Date.parse(found.token.lastUsedAt) >= USED_WRITE_GAP_MS) {
await db.update(apiTokens).set({ lastUsedAt: now.toISOString() }).where(eq(apiTokens.id, found.token.id));
}
return found;
}
/** A token as its owner's list shows it: everything but the hash. */
export function publicApiToken(row: ApiTokenRow, now: Date = new Date()) {
return {
id: row.id,
label: row.label,
tokenPrefix: row.tokenPrefix,
createdAt: row.createdAt,
lastUsedAt: row.lastUsedAt,
expiresAt: row.expiresAt,
expired: row.expiresAt !== null && Date.parse(row.expiresAt) <= now.getTime(),
};
}
export async function listApiTokens(userId: number) {
return db.select().from(apiTokens).where(eq(apiTokens.userId, userId)).orderBy(apiTokens.createdAt);
}
/** Removes one of a person's own tokens. Returns whether there was one. */
export async function removeApiToken(userId: number, id: number): Promise<ApiTokenRow | null> {
const [row] = await db.select().from(apiTokens).where(and(eq(apiTokens.id, id), eq(apiTokens.userId, userId))).limit(1);
if (!row) return null;
await db.delete(apiTokens).where(eq(apiTokens.id, id));
return row;
}
+1
View File
@@ -39,6 +39,7 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
push: "settings",
share_link: "settings",
metrics_token: "settings",
api_token: "settings",
webhook: "settings",
integration: "integrations",
dns: "integrations",
+147 -1
View File
@@ -14,7 +14,8 @@ import tls from "node:tls";
import type { TLSSocket } from "node:tls";
import { and, asc, eq, gte, inArray, lt, sql } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, endpointDaily, endpointResults } from "../db/schema.js";
import { endpointChecks, endpointDaily, endpointHourly, endpointResults } from "../db/schema.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
import { noteRenewal } from "./renewals.js";
import { resolveTarget } from "../utils/networkGuard.js";
@@ -246,6 +247,7 @@ export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c:
.where(eq(endpointChecks.id, c.id));
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
await addToDaily(c.id, iso, result);
await addToHourly(c.id, iso, result);
// A certificate that is now good for much longer than it was has been renewed.
if (result.certExpiresAt) await noteRenewal("certificate", c.name, c.certExpiresAt, result.certExpiresAt, now);
return result;
@@ -272,6 +274,45 @@ async function addToDaily(checkId: number, atIso: string, result: ProbeResult):
}
}
/** How long the hourly figures are kept: between the individual results (two weeks) and the daily figures (years). */
export const KEEP_HOURLY_DAYS = 120;
/** Adds one result to its hour, so an answer-time chart can reach back beyond two weeks with a point an hour rather than one for every check. */
async function addToHourly(checkId: number, atIso: string, result: ProbeResult): Promise<void> {
const timed = result.ok && result.ms !== null;
const ok = result.ok ? 1 : 0;
const ms = timed ? (result.ms as number) : 0;
const hour = `${atIso.slice(0, 13)}:00:00.000Z`;
try {
await db
.insert(endpointHourly)
.values({ checkId, hour, total: 1, ok, msSum: ms, msCount: timed ? 1 : 0, msMax: ms })
.onConflictDoUpdate({
target: [endpointHourly.checkId, endpointHourly.hour],
set: { total: sql`${endpointHourly.total} + 1`, ok: sql`${endpointHourly.ok} + ${ok}`, msSum: sql`${endpointHourly.msSum} + ${ms}`, msCount: sql`${endpointHourly.msCount} + ${timed ? 1 : 0}`, msMax: sql`max(${endpointHourly.msMax}, ${ms})` },
});
} catch (err) {
console.error("[endpoints] couldn't add to the hourly figures:", err);
}
}
const HOURLY_BACKFILL_FLAG = "endpointHourlyBackfilled";
/** The first time after the hourly figures were added, builds them from the results still kept (two weeks). Done once. */
export async function backfillEndpointHourly(): Promise<boolean> {
if (await getInternalFlag(HOURLY_BACKFILL_FLAG)) return false;
await db.run(sql`
INSERT INTO endpoint_hourly (check_id, hour, total, ok, ms_sum, ms_count, ms_max)
SELECT check_id, substr(at, 1, 13) || ':00:00.000Z', count(*), sum(ok), sum(CASE WHEN ok = 1 AND ms IS NOT NULL THEN ms ELSE 0 END),
sum(CASE WHEN ok = 1 AND ms IS NOT NULL THEN 1 ELSE 0 END), coalesce(max(CASE WHEN ok = 1 AND ms IS NOT NULL THEN ms END), 0)
FROM endpoint_results
GROUP BY check_id, substr(at, 1, 13)
ON CONFLICT (check_id, hour) DO UPDATE SET total = excluded.total, ok = excluded.ok, ms_sum = excluded.ms_sum, ms_count = excluded.ms_count, ms_max = excluded.ms_max
`);
await setInternalFlag(HOURLY_BACKFILL_FLAG, new Date().toISOString());
return true;
}
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
@@ -290,6 +331,111 @@ export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRo
export async function pruneResults(now: Date = new Date()): Promise<void> {
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
await db.delete(endpointDaily).where(lt(endpointDaily.day, new Date(now.getTime() - KEEP_DAILY_DAYS * 86_400_000).toISOString().slice(0, 10)));
await db.delete(endpointHourly).where(lt(endpointHourly.hour, new Date(now.getTime() - KEEP_HOURLY_DAYS * 86_400_000).toISOString()));
}
// ─── How a check has done over a long time ─────────────────────────────────
/** A row of figures for some stretch of time (one result, one hour or one day), with the moment it starts at as epoch ms. */
export interface HistoryRow {
t: number;
total: number;
ok: number;
msSum: number;
msCount: number;
msMax: number;
}
export interface HistoryPoint {
/** ISO. */
t: string;
/** Share of the checks in this stretch that answered, 0–100. */
availability: number | null;
/** Average and longest answer time of the good, timed answers; null when there were none. */
avgMs: number | null;
maxMs: number | null;
/** How many checks it stands for. */
checks: number;
}
/** Rows into at most `maxPoints` equal slices of [from, to]: figures are added up per slice, so a slice is as true as the checks in it. */
export function bucketHistory(rows: HistoryRow[], from: number, to: number, maxPoints: number): HistoryPoint[] {
if (rows.length === 0 || to <= from) return [];
const size = Math.max((to - from) / maxPoints, 1);
const buckets = new Map<number, { total: number; ok: number; msSum: number; msCount: number; msMax: number; tSum: number; rows: number }>();
for (const r of rows) {
if (r.t < from || r.t > to || r.total <= 0) continue;
const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size));
const b = buckets.get(key) ?? { total: 0, ok: 0, msSum: 0, msCount: 0, msMax: 0, tSum: 0, rows: 0 };
b.total += r.total;
b.ok += r.ok;
b.msSum += r.msSum;
b.msCount += r.msCount;
b.msMax = Math.max(b.msMax, r.msMax);
b.tSum += r.t;
b.rows += 1;
buckets.set(key, b);
}
return [...buckets.entries()]
.sort((a, b) => a[0] - b[0])
.map(([, b]) => ({
t: new Date(b.tSum / b.rows).toISOString(),
availability: Math.round((b.ok / b.total) * 1000) / 10,
avgMs: b.msCount > 0 ? Math.round(b.msSum / b.msCount) : null,
maxMs: b.msCount > 0 ? b.msMax : null,
checks: b.total,
}));
}
export interface EndpointHistory {
days: number;
/** Which figures it was drawn from: the results themselves (a day or two), the hourly figures, or the daily ones. */
source: "results" | "hourly" | "daily";
points: HistoryPoint[];
/** Over the whole stretch. */
availability: number | null;
avgMs: number | null;
checks: number;
}
/** How a check did over the last `days` (up to a year): availability and answer times, from the finest figures that reach back that far. */
export async function getEndpointHistory(checkId: number, days: number, now: Date = new Date()): Promise<EndpointHistory> {
days = Math.min(365, Math.max(1, Math.round(days)));
const to = now.getTime();
const from = to - days * 86_400_000;
let rows: HistoryRow[];
let source: EndpointHistory["source"];
if (days <= 2) {
source = "results";
const found = await db
.select({ at: endpointResults.at, ok: endpointResults.ok, ms: endpointResults.ms })
.from(endpointResults)
.where(and(eq(endpointResults.checkId, checkId), gte(endpointResults.at, new Date(from).toISOString())));
rows = found.map((r) => {
const timed = r.ok && r.ms !== null;
return { t: Date.parse(r.at), total: 1, ok: r.ok ? 1 : 0, msSum: timed ? (r.ms as number) : 0, msCount: timed ? 1 : 0, msMax: timed ? (r.ms as number) : 0 };
});
} else if (days <= KEEP_HOURLY_DAYS) {
source = "hourly";
const found = await db.select().from(endpointHourly).where(and(eq(endpointHourly.checkId, checkId), gte(endpointHourly.hour, new Date(from).toISOString())));
rows = found.map((r) => ({ t: Date.parse(r.hour), total: r.total, ok: r.ok, msSum: r.msSum, msCount: r.msCount, msMax: r.msMax }));
} else {
source = "daily";
const found = await db.select().from(endpointDaily).where(and(eq(endpointDaily.checkId, checkId), gte(endpointDaily.day, new Date(from).toISOString().slice(0, 10))));
// A day has no record of its slowest answer; its average stands in for it.
rows = found.map((r) => ({ t: Date.parse(`${r.day}T00:00:00.000Z`), total: r.total, ok: r.ok, msSum: r.msSum, msCount: r.msCount, msMax: r.msCount > 0 ? Math.round(r.msSum / r.msCount) : 0 }));
}
const checks = rows.reduce((n, r) => n + r.total, 0);
const okCount = rows.reduce((n, r) => n + r.ok, 0);
const msCount = rows.reduce((n, r) => n + r.msCount, 0);
return {
days,
source,
points: bucketHistory(rows, from, to, days <= 2 ? 96 : days <= KEEP_HOURLY_DAYS ? 200 : 370),
availability: checks === 0 ? null : Math.round((okCount / checks) * 1000) / 10,
avgMs: msCount === 0 ? null : Math.round(rows.reduce((n, r) => n + r.msSum, 0) / msCount),
checks,
};
}
// ─── What the history says ─────────────────────────────────────────────────
+2 -2
View File
@@ -19,7 +19,7 @@ import { getCachedBackupCoverage } from "./backupCoverage.js";
import { lastAttempt as lastSelfBackup, listSnapshots } from "./selfBackup.js";
import { loadAcks, attentionCounts } from "./alertAcks.js";
import { listIncidents } from "./incidents.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
import { computeSecretStatus } from "./secretStatus.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
@@ -132,7 +132,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
try {
for (const s of serverRows) {
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), 3650, 101)) {
for (const f of disksFillingSoon(await getForecastSeries(s.id, now), 3650, 101)) {
out.add("homelab_server_disk_days_to_full", "gauge", "Days until a disk is full at its present rate of filling; only for disks with a steady climb.", f.daysToFull, { server: s.name, mount: f.mount });
}
}
+72
View File
@@ -0,0 +1,72 @@
/**
* The network equipment read from UniFi controllers: devices with their state, address, uptime and what they are connected through, the networks and the Wi-Fi
* names, and which servers are plugged into which device. Shared by the Network devices page and the read API.
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, networkDevices, networkSegments, networkServerLinks, servers } from "../db/schema.js";
import { dependentsOf, loadGraph, needsOfNode, nodeKey } from "./dependencies.js";
import { activeSubjects } from "./maintenance.js";
import { syncStatuses } from "./unifiSync.js";
export async function buildNetworkOverview() {
const [intRows, devices, segments, statuses, graph, silenced, linkRows, serverRows] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "unifi")),
db.select().from(networkDevices),
db.select().from(networkSegments),
syncStatuses(),
loadGraph(),
activeSubjects(),
db.select().from(networkServerLinks),
db.select({ id: servers.id, name: servers.name }).from(servers),
]);
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
const intName = new Map(intRows.map((i) => [i.id, i.name]));
const byExternal = new Map(devices.map((d) => [`${d.integrationId}:${d.externalId}`, d]));
// The servers plugged into each device, by what the controller says is connected (see unifiSync.matchClientsToServers).
const plugged = new Map<number, { serverId: number; serverName: string; port: number | null; wired: boolean }[]>();
const serverLinks: { serverId: number; serverName: string; deviceId: number; deviceName: string; port: number | null; wired: boolean }[] = [];
for (const l of linkRows) {
const device = byExternal.get(`${l.integrationId}:${l.deviceExternalId}`);
const name = serverName.get(l.serverId);
if (!device || !name) continue;
plugged.set(device.id, [...(plugged.get(device.id) ?? []), { serverId: l.serverId, serverName: name, port: l.port, wired: l.wired }]);
serverLinks.push({ serverId: l.serverId, serverName: name, deviceId: device.id, deviceName: device.name, port: l.port, wired: l.wired });
}
return {
serverLinks,
integrations: intRows.map((i) => ({ id: i.id, name: i.name, enabled: i.enabled, status: statuses[i.id] ?? null })),
devices: devices
.map((d) => {
const key = nodeKey("device", d.id);
const uplink = d.uplinkExternalId ? byExternal.get(`${d.integrationId}:${d.uplinkExternalId}`) : undefined;
return {
id: d.id,
integrationId: d.integrationId,
integrationName: intName.get(d.integrationId) ?? "UniFi",
siteName: d.siteName,
name: d.name,
model: d.model,
kind: d.kind,
ip: d.ip,
mac: d.mac,
state: d.state,
uptimeSeconds: d.uptimeSeconds,
firmware: d.firmware,
firmwareUpdatable: d.firmwareUpdatable,
uplink: uplink ? { id: uplink.id, name: uplink.name } : null,
offlineSince: d.offlineSince,
lastOnlineAt: d.lastOnlineAt,
updatedAt: d.updatedAt,
// How many things depend on it (directly or through others), and how many it needs: what the dependency map says about it.
dependents: dependentsOf(graph.edges, key).length,
needs: needsOfNode(graph.edges, key).length,
inMaintenance: silenced.has(key),
servers: (plugged.get(d.id) ?? []).sort((a, b) => a.serverName.localeCompare(b.serverName)),
};
})
.sort((a, b) => a.name.localeCompare(b.name)),
networks: segments.filter((s) => s.kind === "network").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => (a.vlanId ?? 0) - (b.vlanId ?? 0) || a.name.localeCompare(b.name)),
ssids: segments.filter((s) => s.kind === "ssid").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => a.name.localeCompare(b.name)),
};
}
+163
View File
@@ -0,0 +1,163 @@
/**
* What the read API (/api/v1) says. Each of these is a deliberate, stable shape — not the app's own page payloads, which change with the pages — and only things
* any signed-in person can already see: no tokens, no secrets, no passwords, no settings. Dates are ISO 8601 in UTC; sizes are bytes; percentages are 0–100.
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, endpointChecks, servers } from "../db/schema.js";
import { env } from "../env.js";
import { attentionCounts, loadAcks } from "./alertAcks.js";
import { getAlerts } from "./alerts.js";
import { loadStats } from "./endpointChecks.js";
import { describeTarget, listActiveWindows, listUpcomingWindows } from "./maintenance.js";
import { parseHostHealth } from "./hostHealth.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { parseTags } from "./serverTags.js";
import { parseThresholds } from "./serverThresholds.js";
import { getSettings } from "./settingsStore.js";
import { serverStatus } from "./topology.js";
/** An address inside the app as a full address, for a script that has no idea where the app lives. */
export function absoluteUrl(path: string | null): string | null {
return path === null ? null : `${env.appBaseUrl.replace(/\/+$/, "")}${path.startsWith("/") ? "" : "/"}${path}`;
}
const json = <T>(text: string | null, fallback: T): T => {
if (!text) return fallback;
try {
return JSON.parse(text) as T;
} catch {
return fallback;
}
};
type ServerRow = typeof servers.$inferSelect;
export function serverV1(row: ServerRow, offlineMinutes: number, now: Date) {
const status = serverStatus(row, offlineMinutes, now);
const disks = json<{ mount: string; sizeBytes: number; usedBytes: number }[]>(row.disks, []);
const updates = parsePendingUpdates(row.pendingUpdates);
const health = parseHostHealth(row.hostHealth);
return {
id: row.id,
name: row.name,
hostname: row.hostname,
description: row.description,
os: row.osType,
tags: parseTags(row.tags),
ipAddresses: json<string[]>(row.ipAddresses, []),
// "online" while its agent has reported within its limit, "offline" after, "unknown" if it never has.
status: status === "ok" ? "online" : status === "bad" ? "offline" : "unknown",
lastSeenAt: row.lastSeenAt,
createdAt: row.createdAt,
agentVersion: row.agentVersion ?? null,
runsOnServerId: row.parentServerId ?? null,
hardware: { manufacturer: row.hwManufacturer ?? null, model: row.hwModel ?? null, serial: row.hwSerial ?? null },
cpu: { model: row.cpuModel, cores: row.cpuCores, loadPercent: row.cpuLoadPercent },
memory: { totalBytes: row.memTotalBytes, usedBytes: row.memUsedBytes },
disks: disks.map((d) => ({ mount: d.mount, sizeBytes: d.sizeBytes, usedBytes: d.usedBytes, usedPercent: d.sizeBytes > 0 ? Math.round((d.usedBytes / d.sizeBytes) * 1000) / 10 : null })),
updates: updates ? { manager: updates.manager, count: updates.count, security: updates.security ?? null, rebootRequired: !!updates.rebootRequired, reportedAt: updates.reportedAt ?? null } : null,
health: health ? { failedUnits: health.failedUnits?.length ?? 0, reportedAt: health.reportedAt ?? null } : null,
thresholds: parseThresholds(row.thresholds),
url: absoluteUrl(`/servers/${row.id}`),
};
}
export async function serversV1(now: Date = new Date()) {
const [rows, settings] = await Promise.all([db.select().from(servers).orderBy(servers.name), getSettings()]);
return rows.map((r) => serverV1(r, settings.healthChecks.serverOfflineMinutes, now));
}
export async function oneServerV1(id: number, now: Date = new Date()) {
const [row] = await db.select().from(servers).where(eq(servers.id, id)).limit(1);
if (!row) return null;
return serverV1(row, (await getSettings()).healthChecks.serverOfflineMinutes, now);
}
export async function alertsV1() {
const [report, acks] = await Promise.all([getAlerts(false), loadAcks()]);
const attention = attentionCounts(report.alerts, acks);
return {
generatedAt: report.generatedAt,
// What is on the page: those nobody has looked at, by severity, and how many somebody has.
needingAttention: { critical: attention.critical, warning: attention.warning, acknowledged: attention.acknowledged },
counts: report.counts,
couldntCheck: report.couldntCheck,
alerts: report.alerts.map((a) => {
const ack = acks.get(a.id);
return {
id: a.id,
severity: a.severity,
category: a.category,
source: a.source,
message: a.message,
url: absoluteUrl(a.link),
silenced: a.silenced,
flapping: !!a.flapping,
target: a.target,
causedBy: a.causedBy ?? null,
affects: a.affects ?? [],
acknowledgement: ack?.active ? { by: ack.ackedBy, at: ack.ackedAt, note: ack.note, until: ack.until, assignee: ack.assignee?.name ?? null } : null,
};
}),
};
}
export async function inventoryV1() {
const [rows, serverRows] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select({ id: servers.id, name: servers.name }).from(servers)]);
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
return rows.map((a) => ({
id: a.id,
name: a.name,
kind: a.kind,
manufacturer: a.manufacturer,
model: a.model,
serial: a.serial,
location: a.location,
rackUnit: a.rackUnit,
purchaseDate: a.purchaseDate,
purchasePrice: a.purchasePrice,
warrantyUntil: a.warrantyUntil,
endOfSupport: a.endOfSupport,
replacementCost: a.replacementCost,
powerWatts: a.powerWatts,
notes: a.notes,
server: a.serverId !== null && serverName.has(a.serverId) ? { id: a.serverId, name: serverName.get(a.serverId)! } : null,
updatedAt: a.updatedAt,
}));
}
export async function maintenanceV1(now: Date = new Date()) {
const shape = async (w: Awaited<ReturnType<typeof listActiveWindows>>[number]) => {
const target = await describeTarget(w.targetType, w.targetId);
return target ? { id: w.id, target: { type: w.targetType, id: w.targetId, name: target.name, kind: target.kind }, reason: w.reason, startsAt: w.startedAt, endsAt: w.endsAt, createdBy: w.createdBy } : null;
};
const active = (await Promise.all((await listActiveWindows(now)).map(shape))).filter((w): w is NonNullable<typeof w> => w !== null);
const upcoming = (await Promise.all((await listUpcomingWindows(now)).map(shape))).filter((w): w is NonNullable<typeof w> => w !== null);
return { active, upcoming };
}
export async function endpointsV1(now: Date = new Date()) {
const rows = await db.select().from(endpointChecks).orderBy(endpointChecks.name);
const stats = await loadStats(rows.map((r) => r.id), now);
return rows.map((r) => {
const st = stats.get(r.id);
return {
id: r.id,
name: r.name,
kind: r.kind,
target: r.target,
port: r.port,
enabled: r.enabled,
status: !r.enabled ? "paused" : r.lastOk === null ? "unknown" : r.lastOk ? "up" : "down",
lastCheckedAt: r.lastCheckedAt,
lastResponseMs: r.lastResponseMs,
lastMessage: r.lastOk === false ? r.lastMessage : null,
certificateExpiresAt: r.certExpiresAt,
uptime24hPercent: st?.uptime24h ?? null,
uptime7dPercent: st?.uptime7d ?? null,
averageResponseMs24h: st?.avgMs24h ?? null,
serverId: r.serverId,
};
});
}
+174 -19
View File
@@ -1,19 +1,30 @@
/**
* A short history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend
* A history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend
* (and say how soon a filling disk will run out) instead of only the latest figure.
*
* One sample is kept per server every ten minutes at most, however often its agent reports, for 35 days. Everything here is a
* percentage: CPU load (as the agent reports it), memory in use, and each disk's used share.
*
* Beyond those 35 days the same figures are kept coarser: each sample also goes into its hour (kept four months) and its day (kept over two years), as a
* sum, a count, a lowest and a highest. A chart asks for a number of days and is answered from whichever is fine enough and still there: the samples up to
* 35 days, the hours up to 120, the days beyond. A year of a disk filling is then one point a day.
*/
import { and, desc, eq, gte, lt } from "drizzle-orm";
import { and, desc, eq, gte, lt, sql } from "drizzle-orm";
import { db } from "../db/client.js";
import { serverMetrics } from "../db/schema.js";
import { serverMetrics, serverMetricsRollup } from "../db/schema.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
import type { IncomingSystemInfo } from "./taskSync.js";
/** Reports closer together than this add no sample. */
export const MIN_GAP_MS = 10 * 60_000;
export const RETENTION_DAYS = 35;
/** How long the hourly and daily figures are kept. */
export const HOURLY_KEEP_DAYS = 120;
export const DAILY_KEEP_DAYS = 800;
/** The longest stretch a chart may ask for. */
export const MAX_RANGE_DAYS = 365;
const DAY_MS = 24 * 3600_000;
const HOUR_MS = 3600_000;
export interface Sample {
metric: string;
@@ -39,6 +50,35 @@ export function samplesFromSystem(system: IncomingSystemInfo | null | undefined)
return out;
}
/** The start of the hour or day a moment falls in, as ISO. */
export const hourStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / HOUR_MS) * HOUR_MS).toISOString();
export const dayStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / DAY_MS) * DAY_MS).toISOString();
/** Adds samples to their hour and day. Never allowed to break recording the sample itself. */
async function addToRollups(serverId: number, samples: Sample[], at: Date): Promise<void> {
try {
for (const grain of ["h", "d"] as const) {
const bucket = grain === "h" ? hourStart(at.getTime()) : dayStart(at.getTime());
for (const s of samples) {
await db
.insert(serverMetricsRollup)
.values({ serverId, metric: s.metric, grain, at: bucket, sum: s.value, n: 1, min: s.value, max: s.value })
.onConflictDoUpdate({
target: [serverMetricsRollup.serverId, serverMetricsRollup.metric, serverMetricsRollup.grain, serverMetricsRollup.at],
set: {
sum: sql`${serverMetricsRollup.sum} + ${s.value}`,
n: sql`${serverMetricsRollup.n} + 1`,
min: sql`min(${serverMetricsRollup.min}, ${s.value})`,
max: sql`max(${serverMetricsRollup.max}, ${s.value})`,
},
});
}
}
} catch (err) {
console.error("[metrics] couldn't add to the hourly and daily figures:", err);
}
}
/** Records a report's figures, unless this server was sampled a moment ago. Returns whether anything was stored. */
export async function recordServerMetrics(serverId: number, system: IncomingSystemInfo | null | undefined, now: Date = new Date()): Promise<boolean> {
const samples = samplesFromSystem(system);
@@ -54,7 +94,32 @@ export async function recordServerMetrics(serverId: number, system: IncomingSyst
const at = now.toISOString();
await db.insert(serverMetrics).values(samples.map((s) => ({ serverId, metric: s.metric, value: s.value, recordedAt: at })));
await addToRollups(serverId, samples, now);
await db.delete(serverMetrics).where(and(eq(serverMetrics.serverId, serverId), lt(serverMetrics.recordedAt, new Date(now.getTime() - RETENTION_DAYS * DAY_MS).toISOString())));
await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "h"), lt(serverMetricsRollup.at, new Date(now.getTime() - HOURLY_KEEP_DAYS * DAY_MS).toISOString())));
await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "d"), lt(serverMetricsRollup.at, new Date(now.getTime() - DAILY_KEEP_DAYS * DAY_MS).toISOString())));
return true;
}
const BACKFILL_FLAG = "metricsRollupBackfilled";
/**
* The first time after the hourly and daily figures were added, builds them from the samples already kept (up to 35 days), so the longer ranges are not empty
* for weeks. A bucket is written from what the samples say, replacing what is there, so it is right however far it got before. Done once.
*/
export async function backfillMetricRollups(): Promise<boolean> {
if (await getInternalFlag(BACKFILL_FLAG)) return false;
for (const [grain, len] of [["h", 13], ["d", 10]] as const) {
const suffix = grain === "h" ? ":00:00.000Z" : "T00:00:00.000Z";
await db.run(sql`
INSERT INTO server_metrics_rollup (server_id, metric, grain, at, sum, n, min, max)
SELECT server_id, metric, ${grain}, substr(recorded_at, 1, ${len}) || ${suffix}, sum(value), count(*), min(value), max(value)
FROM server_metrics
GROUP BY server_id, metric, substr(recorded_at, 1, ${len})
ON CONFLICT (server_id, metric, grain, at) DO UPDATE SET sum = excluded.sum, n = excluded.n, min = excluded.min, max = excluded.max
`);
}
await setInternalFlag(BACKFILL_FLAG, new Date().toISOString());
return true;
}
@@ -62,6 +127,9 @@ export interface Point {
/** ISO time. */
t: string;
v: number;
/** The lowest and highest figure in what this point stands for, when it stands for several (hourly and daily figures). */
lo?: number;
hi?: number;
}
/** Averages points into at most `maxPoints` equal slices of [from, to], so a month of samples draws as a short line. */
@@ -83,6 +151,42 @@ export function bucketPoints(points: { t: number; v: number }[], from: number, t
.map(([, b]) => ({ t: new Date(b.tSum / b.n).toISOString(), v: round1(b.sum / b.n) }));
}
/** A row of the hourly or daily figures, with the moment it starts at as epoch ms. */
export interface RollupRow {
t: number;
sum: number;
n: number;
min: number;
max: number;
}
/** The average of what each row stands for. */
export function rollupAverages(rows: RollupRow[]): { t: number; v: number }[] {
return rows.filter((r) => r.n > 0).map((r) => ({ t: r.t, v: r.sum / r.n }));
}
/** Rows into at most `maxPoints` slices of [from, to]: the average of the samples in each (not of the rows), with the lowest and highest. */
export function bucketRollups(rows: RollupRow[], from: number, to: number, maxPoints: number): Point[] {
if (rows.length === 0 || to <= from) return [];
const size = Math.max((to - from) / maxPoints, 1);
const buckets = new Map<number, { sum: number; n: number; lo: number; hi: number; tSum: number; rows: number }>();
for (const r of rows) {
if (r.t < from || r.t > to || r.n <= 0) continue;
const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size));
const b = buckets.get(key) ?? { sum: 0, n: 0, lo: Infinity, hi: -Infinity, tSum: 0, rows: 0 };
b.sum += r.sum;
b.n += r.n;
b.lo = Math.min(b.lo, r.min);
b.hi = Math.max(b.hi, r.max);
b.tSum += r.t;
b.rows += 1;
buckets.set(key, b);
}
return [...buckets.entries()]
.sort((a, b) => a[0] - b[0])
.map(([, b]) => ({ t: new Date(b.tSum / b.rows).toISOString(), v: round1(b.sum / b.n), lo: round1(b.lo), hi: round1(b.hi) }));
}
export interface Trend {
/** Change in the percentage per day, from a straight line through the points. */
perDay: number;
@@ -175,35 +279,86 @@ function describe(metric: string): { kind: MetricSeries["kind"]; label: string }
const ORDER = { cpu: 0, mem: 1, disk: 2 } as const;
/** Every series recorded for a server over the last `days`, ready to draw. */
/** Which of the kept figures answers a stretch of `days`: the samples while they reach back that far, then the hours, then the days. */
export function sourceFor(days: number): "samples" | "hourly" | "daily" {
return days <= RETENTION_DAYS ? "samples" : days <= HOURLY_KEEP_DAYS ? "hourly" : "daily";
}
/** How many points a chart over `days` is drawn from at most. */
const pointsFor = (days: number) => (days <= RETENTION_DAYS ? 96 : days <= HOURLY_KEEP_DAYS ? 200 : MAX_RANGE_DAYS + 5);
/** The latest sample of each figure of a server, whatever range is asked for. */
async function latestSamples(serverId: number): Promise<Map<string, number>> {
const rows = await db.all<{ metric: string; value: number }>(
sql`SELECT metric, value FROM server_metrics WHERE id IN (SELECT max(id) FROM server_metrics WHERE server_id = ${serverId} GROUP BY metric)`,
);
return new Map(rows.map((r) => [r.metric, r.value]));
}
/** Every series recorded for a server over the last `days` (up to a year), ready to draw. */
export async function getServerMetrics(serverId: number, days: number, now: Date = new Date()): Promise<MetricSeries[]> {
days = Math.min(MAX_RANGE_DAYS, Math.max(1, days));
const to = now.getTime();
const from = to - days * DAY_MS;
const rows = await db
.select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt })
.from(serverMetrics)
.where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString())))
.orderBy(serverMetrics.recordedAt);
const source = sourceFor(days);
const byMetric = new Map<string, { t: number; v: number }[]>();
for (const r of rows) {
const list = byMetric.get(r.metric) ?? [];
list.push({ t: Date.parse(r.at), v: r.value });
byMetric.set(r.metric, list);
// The figures, per metric, as { t, v } for a trend and as points for the chart.
const raw = new Map<string, { t: number; v: number }[]>();
const points = new Map<string, Point[]>();
if (source === "samples") {
const rows = await db
.select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt })
.from(serverMetrics)
.where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString())))
.orderBy(serverMetrics.recordedAt);
for (const r of rows) raw.set(r.metric, [...(raw.get(r.metric) ?? []), { t: Date.parse(r.at), v: r.value }]);
for (const [metric, list] of raw) points.set(metric, bucketPoints(list, from, to, pointsFor(days)));
} else {
const rows = await db
.select()
.from(serverMetricsRollup)
.where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, source === "hourly" ? "h" : "d"), gte(serverMetricsRollup.at, new Date(from).toISOString())))
.orderBy(serverMetricsRollup.at);
const byMetric = new Map<string, RollupRow[]>();
for (const r of rows) byMetric.set(r.metric, [...(byMetric.get(r.metric) ?? []), { t: Date.parse(r.at), sum: r.sum, n: r.n, min: r.min, max: r.max }]);
for (const [metric, list] of byMetric) {
raw.set(metric, rollupAverages(list));
points.set(metric, bucketRollups(list, from, to, pointsFor(days)));
}
}
const latest = source === "samples" ? null : await latestSamples(serverId);
return [...byMetric.entries()]
.map(([metric, raw]) => {
return [...raw.entries()]
.map(([metric, list]) => {
const { kind, label } = describe(metric);
const newest = latest?.get(metric) ?? list[list.length - 1]?.v;
return {
metric,
kind,
label,
points: bucketPoints(raw, from, to, 96),
latest: raw[raw.length - 1]?.v ?? null,
points: points.get(metric) ?? [],
latest: newest === undefined ? null : source === "samples" ? newest : round1(newest),
// A disk filling is the thing worth forecasting; load and memory go up and down by the hour.
trend: kind === "disk" ? trendOf(raw) : null,
trend: kind === "disk" ? trendOf(list) : null,
};
})
.sort((a, b) => ORDER[a.kind] - ORDER[b.kind] || a.label.localeCompare(b.label));
}
/** How far back the longer look at a disk goes when the last two weeks say it is flat. */
export const LONG_FORECAST_DAYS = 90;
/**
* The series to forecast from: the last two weeks, as before — except that a disk those two weeks call flat is looked at again over the last
* ninety days, so a slow, steady climb (a point a week) is seen too. The longer trend replaces the short one only if it is really rising.
*/
export async function getForecastSeries(serverId: number, now: Date = new Date()): Promise<MetricSeries[]> {
const short = await getServerMetrics(serverId, 14, now);
if (!short.some((s) => s.kind === "disk" && (!s.trend || s.trend.daysToFull === null))) return short;
const long = await getServerMetrics(serverId, LONG_FORECAST_DAYS, now);
return short.map((s) => {
if (s.kind !== "disk" || (s.trend && s.trend.daysToFull !== null)) return s;
const longer = long.find((l) => l.metric === s.metric);
return longer?.trend && longer.trend.daysToFull !== null ? { ...s, trend: longer.trend } : s;
});
}
+2 -2
View File
@@ -16,7 +16,7 @@ import { describeTarget, listUpcomingWindows } from "./maintenance.js";
import { notify, notifyNow } from "./notify.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { computeSecretStatus } from "./secretStatus.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
@@ -163,7 +163,7 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
const own = parseThresholds(s.thresholds);
const forecastDays = effectiveThresholds(settings.healthChecks, own).diskForecastDays;
if (!(forecastDays > 0)) continue;
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
for (const f of disksFillingSoon(await getForecastSeries(s.id, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
}
}