A year of history, one chart, and a read API with personal tokens
History: server load, memory and disks are also kept as hourly (four months) and daily (over two years) averages with lowest and highest; endpoint answers as hourly (four months) beside the daily figures. Charts ask for 24 hours to a year and are drawn from whichever is fine enough. A disk that two weeks call flat is looked at again over ninety days for the forecast. Built once from what is kept. One chart: TimeChart replaces the sparkline and the multi-line chart: hover and arrow-key readout, a lowest/highest band, gaps shown as gaps, the same ranges everywhere and the choice remembered. Used for server trends, comparing servers, a check's availability and answer times (Endpoints > History) and a server's availability. Read API: /api/v1 (servers, metrics, alerts, incidents, inventory, network, endpoints and their history, maintenance) with personal read-only tokens made under the user menu: shown once, hashed, 30/90/365 days or none, ten at most, removed with their owner. GET only. Documented in API.md. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5e26e77340
commit
68b41b5586
47 files changed
+12360
-247
No files matched your search
@@ -47,6 +47,8 @@ import { alertsRouter } from "./routes/alerts.js";
|
||||
import { mapRouter } from "./routes/map.js";
|
||||
import { networkRouter } from "./routes/network.js";
|
||||
import { myWorkRouter } from "./routes/myWork.js";
|
||||
import { apiTokensRouter } from "./routes/apiTokens.js";
|
||||
import { publicApiRouter } from "./routes/publicApi.js";
|
||||
import { generatorRouter } from "./routes/generator.js";
|
||||
import { alertTicketsRouter } from "./routes/alertTickets.js";
|
||||
import { avatarsRouter } from "./routes/avatars.js";
|
||||
@@ -83,6 +85,8 @@ export function createApp() {
|
||||
app.use("/api/agent/report", agentReportRouter);
|
||||
// A job reporting that it ran: a token in the address, no session, and no browser to be checked for where it came from.
|
||||
app.use("/api/heartbeat", heartbeatPingRouter);
|
||||
// The read API: a personal token in a header, GET only, no session and nothing for the same-origin check to protect.
|
||||
app.use("/api/v1", publicApiRouter);
|
||||
// Everything after this is reached with a session cookie, so a request that came from another origin is refused here.
|
||||
app.use(sameOriginGuard);
|
||||
app.use(express.json());
|
||||
@@ -114,6 +118,7 @@ export function createApp() {
|
||||
}
|
||||
|
||||
app.use("/auth", authRouter);
|
||||
app.use("/api/me/tokens", apiTokensRouter);
|
||||
app.use("/api/me", meRouter);
|
||||
app.use("/api/avatars", avatarsRouter);
|
||||
app.use("/api/heartbeats", heartbeatsRouter);
|
||||
|
||||
@@ -263,6 +263,26 @@ export const serverMetrics = sqliteTable(
|
||||
(table) => [index("server_metrics_lookup_idx").on(table.serverId, table.metric, table.recordedAt)],
|
||||
);
|
||||
|
||||
// The same figures, kept longer and coarser: one row per server, figure and hour (grain "h", kept a few months) or day (grain "d", kept for years), with what is
|
||||
// needed to draw them — the sum and count (so the average can be worked out as more samples arrive), and the lowest and highest. `at` is the start of the hour
|
||||
// or day, as ISO. Written as each sample is recorded; see services/serverMetrics.ts.
|
||||
export const serverMetricsRollup = sqliteTable(
|
||||
"server_metrics_rollup",
|
||||
{
|
||||
serverId: integer("server_id")
|
||||
.notNull()
|
||||
.references(() => servers.id, { onDelete: "cascade" }),
|
||||
metric: text("metric").notNull(),
|
||||
grain: text("grain").$type<"h" | "d">().notNull(),
|
||||
at: text("at").notNull(),
|
||||
sum: real("sum").notNull(),
|
||||
n: integer("n").notNull(),
|
||||
min: real("min").notNull(),
|
||||
max: real("max").notNull(),
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.serverId, t.metric, t.grain, t.at] })],
|
||||
);
|
||||
|
||||
// ─── Settings (key/value) ───────────────────────────────────────────────────
|
||||
|
||||
export const settings = sqliteTable("settings", {
|
||||
@@ -993,6 +1013,24 @@ export const endpointDaily = sqliteTable(
|
||||
(t) => [primaryKey({ columns: [t.checkId, t.day] })],
|
||||
);
|
||||
|
||||
// The same, by the hour: how many times a check was asked in that hour, how many answered, and the answer times — kept a few months, between the individual
|
||||
// results (two weeks) and the daily figures (years), so an answer-time chart can reach back beyond two weeks without a point for every check.
|
||||
export const endpointHourly = sqliteTable(
|
||||
"endpoint_hourly",
|
||||
{
|
||||
checkId: integer("check_id")
|
||||
.notNull()
|
||||
.references(() => endpointChecks.id, { onDelete: "cascade" }),
|
||||
hour: text("hour").notNull(), // ISO, the start of the hour (UTC)
|
||||
total: integer("total").notNull().default(0),
|
||||
ok: integer("ok").notNull().default(0),
|
||||
msSum: integer("ms_sum").notNull().default(0),
|
||||
msCount: integer("ms_count").notNull().default(0),
|
||||
msMax: integer("ms_max").notNull().default(0),
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.checkId, t.hour] })],
|
||||
);
|
||||
|
||||
// A certificate, domain registration or secret whose expiry moved later: it was renewed. Written when the app sees it happen, so it only goes back
|
||||
// as far as the version that began watching (services/renewals.ts).
|
||||
export const renewals = sqliteTable(
|
||||
@@ -1050,6 +1088,25 @@ export const metricsTokens = sqliteTable("metrics_tokens", {
|
||||
lastUsedAt: text("last_used_at"), // ISO
|
||||
});
|
||||
|
||||
// ─── Personal API tokens: read-only access to /api/v1 for scripts and Home Assistant ─────
|
||||
|
||||
// One person's own: made by them, shown to them once, and gone when they are. Only a hash of the secret is kept. A token only ever reads (the API has no other verbs),
|
||||
// and sees what any signed-in person sees.
|
||||
export const apiTokens = sqliteTable("api_tokens", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
userId: integer("user_id")
|
||||
.notNull()
|
||||
.references(() => users.id, { onDelete: "cascade" }),
|
||||
label: text("label").notNull(), // e.g. "Home Assistant"
|
||||
tokenHash: text("token_hash").notNull().unique(),
|
||||
tokenPrefix: text("token_prefix").notNull(),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
lastUsedAt: text("last_used_at"), // ISO
|
||||
expiresAt: text("expires_at"), // ISO; null = until removed
|
||||
});
|
||||
|
||||
// ─── Event webhooks: tell another system when something happens ─────────────
|
||||
|
||||
export const webhookEvents = ["alert.opened", "alert.resolved", "maintenance.started", "maintenance.ended"] as const;
|
||||
|
||||
@@ -25,6 +25,8 @@ import { initEndpointMonitor } from "./services/endpointMonitor.js";
|
||||
import { initPatchPlans } from "./services/patchPlans.js";
|
||||
import { initFlapMonitor } from "./services/flapMonitor.js";
|
||||
import { initUnifiSync } from "./services/unifiSync.js";
|
||||
import { backfillMetricRollups } from "./services/serverMetrics.js";
|
||||
import { backfillEndpointHourly } from "./services/endpointChecks.js";
|
||||
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
|
||||
import { initIncidentRecorder } from "./services/incidents.js";
|
||||
import { getAlerts } from "./services/alerts.js";
|
||||
@@ -69,6 +71,8 @@ initEndpointMonitor();
|
||||
initPatchPlans();
|
||||
initFlapMonitor();
|
||||
initUnifiSync();
|
||||
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
|
||||
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
|
||||
|
||||
createApp().listen(env.port, () => {
|
||||
console.log(`homelab-manager listening on port ${env.port}`);
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { Router } from "express";
|
||||
import { z } from "zod";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { createApiToken, EXPIRY_CHOICES, listApiTokens, MAX_TOKENS_PER_PERSON, publicApiToken, removeApiToken, TokenLimitError } from "../services/apiTokens.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
/** A person's own API tokens: anyone signed in makes, sees and removes theirs — never anybody else's. */
|
||||
export const apiTokensRouter = Router();
|
||||
apiTokensRouter.use(requireAuth);
|
||||
|
||||
const createSchema = z.object({
|
||||
label: z.string().trim().min(1).max(60),
|
||||
// Days until it stops working; null (or left out) for until it is removed.
|
||||
expiresInDays: z.union([z.literal(30), z.literal(90), z.literal(365)]).nullable().optional(),
|
||||
});
|
||||
|
||||
apiTokensRouter.get("/", asyncHandler(async (req, res) => {
|
||||
const rows = await listApiTokens(req.currentUser!.id);
|
||||
res.json({ tokens: rows.map((r) => publicApiToken(r)), max: MAX_TOKENS_PER_PERSON, expiryChoices: EXPIRY_CHOICES });
|
||||
}));
|
||||
|
||||
// The secret is in this answer and nowhere else, ever.
|
||||
apiTokensRouter.post("/", asyncHandler(async (req, res) => {
|
||||
const parsed = createSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give the token a name, like Home Assistant, and choose 30, 90 or 365 days or no end." });
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
const { row, token } = await createApiToken({ userId: actor.id, label: parsed.data.label, expiresInDays: parsed.data.expiresInDays ?? null });
|
||||
await recordAudit({ actor, category: "api_token", action: "create", targetType: "api_token", targetId: row.id, detail: { label: row.label, expiresAt: row.expiresAt } });
|
||||
res.status(201).json({ token: publicApiToken(row), secret: token });
|
||||
} catch (err) {
|
||||
if (err instanceof TokenLimitError) return res.status(400).json({ error: "too_many", message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
apiTokensRouter.delete("/:id", asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const actor = req.currentUser!;
|
||||
const removed = await removeApiToken(actor.id, id);
|
||||
if (!removed) return res.status(404).json({ error: "not_found" });
|
||||
await recordAudit({ actor, category: "api_token", action: "delete", targetType: "api_token", targetId: id, detail: { label: removed.label } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -6,7 +6,7 @@ import { endpointChecks, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { CHECK_KINDS, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
|
||||
import { CHECK_KINDS, getEndpointHistory, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
|
||||
import { loadDismissed, loadSuggestions, saveDismissed } from "../services/endpointSuggestions.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
@@ -63,6 +63,17 @@ endpointsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
});
|
||||
}));
|
||||
|
||||
// How one check has done over a day, a week, a month, three months or a year: availability and answer times, for the chart on its page.
|
||||
endpointsRouter.get("/:id/history", asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [check] = await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
|
||||
if (!check) return res.status(404).json({ error: "not_found" });
|
||||
const requested = Number(req.query.days);
|
||||
const days = Number.isFinite(requested) ? Math.min(365, Math.max(1, Math.round(requested))) : 7;
|
||||
res.json({ check, ...(await getEndpointHistory(id, days)) });
|
||||
}));
|
||||
|
||||
function valuesOf(d: z.infer<typeof checkSchema>) {
|
||||
return {
|
||||
name: d.name,
|
||||
|
||||
@@ -2,12 +2,11 @@ import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, networkDevices, networkSegments, networkServerLinks, servers } from "../db/schema.js";
|
||||
import { integrations } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { dependentsOf, loadGraph, needsOfNode, nodeKey } from "../services/dependencies.js";
|
||||
import { activeSubjects } from "../services/maintenance.js";
|
||||
import { syncAllUnifi, syncStatuses, syncUnifiIntegration } from "../services/unifiSync.js";
|
||||
import { buildNetworkOverview } from "../services/networkOverview.js";
|
||||
import { syncAllUnifi, syncUnifiIntegration } from "../services/unifiSync.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const networkRouter = Router();
|
||||
@@ -19,65 +18,7 @@ networkRouter.use(requireAuth);
|
||||
* names. Everyone signed in can see it (it is the same picture the dependency map and the alerts draw on); only starting a fresh read needs more.
|
||||
*/
|
||||
networkRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [intRows, devices, segments, statuses, graph, silenced, linkRows, serverRows] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "unifi")),
|
||||
db.select().from(networkDevices),
|
||||
db.select().from(networkSegments),
|
||||
syncStatuses(),
|
||||
loadGraph(),
|
||||
activeSubjects(),
|
||||
db.select().from(networkServerLinks),
|
||||
db.select({ id: servers.id, name: servers.name }).from(servers),
|
||||
]);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
const intName = new Map(intRows.map((i) => [i.id, i.name]));
|
||||
const byExternal = new Map(devices.map((d) => [`${d.integrationId}:${d.externalId}`, d]));
|
||||
// The servers plugged into each device, by what the controller says is connected (see unifiSync.matchClientsToServers).
|
||||
const plugged = new Map<number, { serverId: number; serverName: string; port: number | null; wired: boolean }[]>();
|
||||
const serverLinks: { serverId: number; serverName: string; deviceId: number; deviceName: string; port: number | null; wired: boolean }[] = [];
|
||||
for (const l of linkRows) {
|
||||
const device = byExternal.get(`${l.integrationId}:${l.deviceExternalId}`);
|
||||
const name = serverName.get(l.serverId);
|
||||
if (!device || !name) continue;
|
||||
plugged.set(device.id, [...(plugged.get(device.id) ?? []), { serverId: l.serverId, serverName: name, port: l.port, wired: l.wired }]);
|
||||
serverLinks.push({ serverId: l.serverId, serverName: name, deviceId: device.id, deviceName: device.name, port: l.port, wired: l.wired });
|
||||
}
|
||||
res.json({
|
||||
serverLinks,
|
||||
integrations: intRows.map((i) => ({ id: i.id, name: i.name, enabled: i.enabled, status: statuses[i.id] ?? null })),
|
||||
devices: devices
|
||||
.map((d) => {
|
||||
const key = nodeKey("device", d.id);
|
||||
const uplink = d.uplinkExternalId ? byExternal.get(`${d.integrationId}:${d.uplinkExternalId}`) : undefined;
|
||||
return {
|
||||
id: d.id,
|
||||
integrationId: d.integrationId,
|
||||
integrationName: intName.get(d.integrationId) ?? "UniFi",
|
||||
siteName: d.siteName,
|
||||
name: d.name,
|
||||
model: d.model,
|
||||
kind: d.kind,
|
||||
ip: d.ip,
|
||||
mac: d.mac,
|
||||
state: d.state,
|
||||
uptimeSeconds: d.uptimeSeconds,
|
||||
firmware: d.firmware,
|
||||
firmwareUpdatable: d.firmwareUpdatable,
|
||||
uplink: uplink ? { id: uplink.id, name: uplink.name } : null,
|
||||
offlineSince: d.offlineSince,
|
||||
lastOnlineAt: d.lastOnlineAt,
|
||||
updatedAt: d.updatedAt,
|
||||
// How many things depend on it (directly or through others), and how many it needs: what the dependency map says about it.
|
||||
dependents: dependentsOf(graph.edges, key).length,
|
||||
needs: needsOfNode(graph.edges, key).length,
|
||||
inMaintenance: silenced.has(key),
|
||||
servers: (plugged.get(d.id) ?? []).sort((a, b) => a.serverName.localeCompare(b.serverName)),
|
||||
};
|
||||
})
|
||||
.sort((a, b) => a.name.localeCompare(b.name)),
|
||||
networks: segments.filter((s) => s.kind === "network").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => (a.vlanId ?? 0) - (b.vlanId ?? 0) || a.name.localeCompare(b.name)),
|
||||
ssids: segments.filter((s) => s.kind === "ssid").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
});
|
||||
res.json(await buildNetworkOverview());
|
||||
}));
|
||||
|
||||
const syncSchema = z.object({ integrationId: z.number().int().positive().optional() });
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
import { Router, type NextFunction, type Request, type Response } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, servers } from "../db/schema.js";
|
||||
import { resolveApiToken, type ApiTokenRow, type ApiTokenUser } from "../services/apiTokens.js";
|
||||
import { getEndpointHistory } from "../services/endpointChecks.js";
|
||||
import { getIncident, listIncidents } from "../services/incidents.js";
|
||||
import { buildNetworkOverview } from "../services/networkOverview.js";
|
||||
import { alertsV1, endpointsV1, inventoryV1, maintenanceV1, oneServerV1, serversV1 } from "../services/publicApi.js";
|
||||
import { getServerMetrics, MAX_RANGE_DAYS } from "../services/serverMetrics.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { rateLimit } from "../utils/rateLimit.js";
|
||||
|
||||
/**
|
||||
* The read API: JSON about servers, alerts, incidents, inventory, network equipment, endpoint checks and maintenance, for scripts and things like Home Assistant.
|
||||
* A personal token in the Authorization header (never the address, which ends up in logs); GET only; and nothing here writes anything.
|
||||
*/
|
||||
export const publicApiRouter = Router();
|
||||
|
||||
declare module "express-serve-static-core" {
|
||||
interface Request {
|
||||
apiToken?: { token: ApiTokenRow; user: ApiTokenUser };
|
||||
}
|
||||
}
|
||||
|
||||
const limiter = rateLimit({ windowMs: 60_000, max: 300, message: "Too many requests." });
|
||||
|
||||
publicApiRouter.use(limiter);
|
||||
publicApiRouter.use((req: Request, res: Response, next: NextFunction) => {
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
if (req.method !== "GET" && req.method !== "HEAD") {
|
||||
res.setHeader("Allow", "GET, HEAD");
|
||||
return res.status(405).json({ error: "read_only", message: "This API only reads. It has no way to change anything." });
|
||||
}
|
||||
next();
|
||||
});
|
||||
|
||||
publicApiRouter.use(
|
||||
asyncHandler(async (req, res, next) => {
|
||||
const match = /^Bearer\s+(\S+)$/i.exec(req.get("authorization") ?? "");
|
||||
const found = match ? await resolveApiToken(match[1]) : null;
|
||||
if (!found) {
|
||||
// The same answer for no token, a wrong one, an expired one and a removed one.
|
||||
res.setHeader("WWW-Authenticate", 'Bearer realm="homelab-manager"');
|
||||
return res.status(401).json({ error: "unauthorized", message: "A token is needed: send it as 'Authorization: Bearer <token>'. Make one from the user menu → API tokens." });
|
||||
}
|
||||
req.apiToken = found;
|
||||
next();
|
||||
}),
|
||||
);
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
const daysOf = (v: unknown, fallback: number) => {
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? Math.min(MAX_RANGE_DAYS, Math.max(1, Math.round(n))) : fallback;
|
||||
};
|
||||
|
||||
publicApiRouter.get("/", (_req, res) => {
|
||||
res.json({
|
||||
name: "Homelab Manager read API",
|
||||
version: 1,
|
||||
endpoints: [
|
||||
"GET /api/v1/me",
|
||||
"GET /api/v1/servers?tag=&status=",
|
||||
"GET /api/v1/servers/:id",
|
||||
"GET /api/v1/servers/:id/metrics?days=",
|
||||
"GET /api/v1/alerts?severity=&category=&acknowledged=&silenced=",
|
||||
"GET /api/v1/incidents?status=&limit=",
|
||||
"GET /api/v1/incidents/:id",
|
||||
"GET /api/v1/inventory",
|
||||
"GET /api/v1/network",
|
||||
"GET /api/v1/endpoints",
|
||||
"GET /api/v1/endpoints/:id/history?days=",
|
||||
"GET /api/v1/maintenance",
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
publicApiRouter.get("/me", (req, res) => {
|
||||
const { token, user } = req.apiToken!;
|
||||
res.json({ user: { id: user.id, name: user.name, email: user.email, role: user.role }, token: { id: token.id, label: token.label, expiresAt: token.expiresAt } });
|
||||
});
|
||||
|
||||
publicApiRouter.get("/servers", asyncHandler(async (req, res) => {
|
||||
let list = await serversV1();
|
||||
if (typeof req.query.tag === "string" && req.query.tag) {
|
||||
const tag = req.query.tag.toLowerCase();
|
||||
list = list.filter((s) => s.tags.includes(tag));
|
||||
}
|
||||
if (req.query.status === "online" || req.query.status === "offline" || req.query.status === "unknown") list = list.filter((s) => s.status === req.query.status);
|
||||
res.json({ servers: list });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/servers/:id", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const server = await oneServerV1(id);
|
||||
if (!server) return res.status(404).json({ error: "not_found" });
|
||||
res.json({ server });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/servers/:id/metrics", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1);
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
const days = daysOf(req.query.days, 7);
|
||||
res.json({ days, series: await getServerMetrics(id, days) });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/alerts", asyncHandler(async (req, res) => {
|
||||
const report = await alertsV1();
|
||||
let alerts = report.alerts;
|
||||
const q = req.query;
|
||||
if (typeof q.severity === "string") alerts = alerts.filter((a) => a.severity === q.severity);
|
||||
if (typeof q.category === "string") alerts = alerts.filter((a) => a.category === q.category);
|
||||
if (q.acknowledged === "true" || q.acknowledged === "false") alerts = alerts.filter((a) => (a.acknowledgement !== null) === (q.acknowledged === "true"));
|
||||
if (q.silenced === "true" || q.silenced === "false") alerts = alerts.filter((a) => a.silenced === (q.silenced === "true"));
|
||||
res.json({ ...report, alerts });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/incidents", asyncHandler(async (req, res) => {
|
||||
const limit = Math.min(500, Math.max(1, Math.round(Number(req.query.limit)) || 100));
|
||||
let list = await listIncidents({ limit });
|
||||
if (req.query.status === "open") list = list.filter((i) => i.open);
|
||||
else if (req.query.status === "resolved") list = list.filter((i) => !i.open);
|
||||
res.json({ incidents: list });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/incidents/:id", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const detail = await getIncident(id);
|
||||
if (!detail) return res.status(404).json({ error: "not_found" });
|
||||
res.json(detail);
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/inventory", asyncHandler(async (_req, res) => {
|
||||
res.json({ assets: await inventoryV1() });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/network", asyncHandler(async (_req, res) => {
|
||||
const overview = await buildNetworkOverview();
|
||||
res.json({ devices: overview.devices, networks: overview.networks, ssids: overview.ssids, serverLinks: overview.serverLinks });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/endpoints", asyncHandler(async (_req, res) => {
|
||||
res.json({ endpoints: await endpointsV1() });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/endpoints/:id/history", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [check] = await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
|
||||
if (!check) return res.status(404).json({ error: "not_found" });
|
||||
res.json({ check, ...(await getEndpointHistory(id, daysOf(req.query.days, 7))) });
|
||||
}));
|
||||
|
||||
publicApiRouter.get("/maintenance", asyncHandler(async (_req, res) => {
|
||||
res.json(await maintenanceV1());
|
||||
}));
|
||||
|
||||
publicApiRouter.use((_req, res) => {
|
||||
res.status(404).json({ error: "not_found", message: "No such address. GET /api/v1 lists them." });
|
||||
});
|
||||
@@ -12,7 +12,7 @@ import { markIgnoredServices, parseHostHealth } from "../services/hostHealth.js"
|
||||
import { describeDiagnostics, parseDiagnostics } from "../services/agentDiagnostics.js";
|
||||
import { checkThresholds, parseThresholds } from "../services/serverThresholds.js";
|
||||
import { latestAgentVersions } from "../services/agentVersions.js";
|
||||
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
|
||||
import { getServerMetrics, MAX_RANGE_DAYS } from "../services/serverMetrics.js";
|
||||
import { getServerActivity } from "../services/serverActivity.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { ID_PATTERNS } from "../utils/pathSegment.js";
|
||||
@@ -272,7 +272,7 @@ serversRouter.get("/:id/metrics", asyncHandler(async (req, res) => {
|
||||
const [server] = await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1);
|
||||
if (!server) return res.status(404).json({ error: "not_found" });
|
||||
const requested = Number(req.query.days);
|
||||
const days = Number.isFinite(requested) ? Math.min(RETENTION_DAYS, Math.max(1, Math.round(requested))) : 7;
|
||||
const days = Number.isFinite(requested) ? Math.min(MAX_RANGE_DAYS, Math.max(1, Math.round(requested))) : 7;
|
||||
res.json({ days, series: await getServerMetrics(id, days) });
|
||||
}));
|
||||
|
||||
|
||||
@@ -6,6 +6,9 @@ import { getUptime } from "../services/uptimeHistory.js";
|
||||
export const uptimeRouter = Router();
|
||||
|
||||
// Anyone signed in who can see the server and integration lists can see how those have been doing.
|
||||
uptimeRouter.get("/", requireAuth, asyncHandler(async (_req, res) => {
|
||||
res.json(await getUptime());
|
||||
// `?days=` is 30 by default and at most a year (as far back as the audit log it is worked out from goes).
|
||||
uptimeRouter.get("/", requireAuth, asyncHandler(async (req, res) => {
|
||||
const requested = Number(req.query.days);
|
||||
const days = Number.isFinite(requested) ? Math.min(365, Math.max(1, Math.round(requested))) : undefined;
|
||||
res.json(await getUptime(new Date(), days));
|
||||
}));
|
||||
@@ -12,7 +12,7 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, heartbeats, integrations, networkDevices, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
import { recordFromReport } from "./incidents.js";
|
||||
@@ -499,7 +499,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const own = parseThresholds(s.thresholds);
|
||||
const forecastDays = effectiveThresholds(healthChecks, own).diskForecastDays;
|
||||
if (!(forecastDays > 0)) continue;
|
||||
const series = await getServerMetrics(s.id, 14);
|
||||
const series = await getForecastSeries(s.id);
|
||||
for (const f of disksFillingSoon(series, forecastDays, (mount) => diskLimit(own, mount, healthChecks.diskUsagePercent))) {
|
||||
push({
|
||||
category: "disk",
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* Personal API tokens: read-only access to /api/v1 for scripts, Home Assistant and the like. Each belongs to the person who made it — they see and remove their own —
|
||||
* and goes when they do. Like an agent token or a share link, the secret is shown once, when it is made, and only its SHA-256 is kept.
|
||||
*/
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { apiTokens, users } from "../db/schema.js";
|
||||
|
||||
export type ApiTokenRow = typeof apiTokens.$inferSelect;
|
||||
export type ApiTokenUser = typeof users.$inferSelect;
|
||||
|
||||
const hash = (token: string) => createHash("sha256").update(token).digest("hex");
|
||||
|
||||
/** Most tokens one person may have: one for each thing that reads, and a spare. */
|
||||
export const MAX_TOKENS_PER_PERSON = 10;
|
||||
/** How long a token may be asked to last, in days (or never). */
|
||||
export const EXPIRY_CHOICES = [30, 90, 365] as const;
|
||||
|
||||
/** Use is not recorded more than once in this time, so a script that reads every few seconds doesn't write the database each time. */
|
||||
const USED_WRITE_GAP_MS = 60_000;
|
||||
|
||||
export class TokenLimitError extends Error {}
|
||||
|
||||
export async function createApiToken(params: { userId: number; label: string; expiresInDays: number | null }, now: Date = new Date()): Promise<{ row: ApiTokenRow; token: string }> {
|
||||
const existing = await db.select({ id: apiTokens.id }).from(apiTokens).where(eq(apiTokens.userId, params.userId));
|
||||
if (existing.length >= MAX_TOKENS_PER_PERSON) throw new TokenLimitError(`There are already ${MAX_TOKENS_PER_PERSON} tokens. Remove one you don't use first.`);
|
||||
const token = `hlm_p_${randomBytes(24).toString("base64url")}`;
|
||||
const expiresAt = params.expiresInDays === null ? null : new Date(now.getTime() + params.expiresInDays * 86_400_000).toISOString();
|
||||
const [row] = await db.insert(apiTokens).values({ userId: params.userId, label: params.label, tokenHash: hash(token), tokenPrefix: token.slice(0, 12), expiresAt }).returning();
|
||||
return { row, token };
|
||||
}
|
||||
|
||||
/** The token a secret belongs to and whose it is, if it is one that still works; notes that it was used. */
|
||||
export async function resolveApiToken(secret: string, now: Date = new Date()): Promise<{ token: ApiTokenRow; user: ApiTokenUser } | null> {
|
||||
if (typeof secret !== "string" || secret.length < 20 || secret.length > 128) return null;
|
||||
const [found] = await db
|
||||
.select({ token: apiTokens, user: users })
|
||||
.from(apiTokens)
|
||||
.innerJoin(users, eq(users.id, apiTokens.userId))
|
||||
.where(eq(apiTokens.tokenHash, hash(secret)))
|
||||
.limit(1);
|
||||
if (!found) return null;
|
||||
if (found.token.expiresAt && Date.parse(found.token.expiresAt) <= now.getTime()) return null;
|
||||
if (!found.token.lastUsedAt || now.getTime() - Date.parse(found.token.lastUsedAt) >= USED_WRITE_GAP_MS) {
|
||||
await db.update(apiTokens).set({ lastUsedAt: now.toISOString() }).where(eq(apiTokens.id, found.token.id));
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/** A token as its owner's list shows it: everything but the hash. */
|
||||
export function publicApiToken(row: ApiTokenRow, now: Date = new Date()) {
|
||||
return {
|
||||
id: row.id,
|
||||
label: row.label,
|
||||
tokenPrefix: row.tokenPrefix,
|
||||
createdAt: row.createdAt,
|
||||
lastUsedAt: row.lastUsedAt,
|
||||
expiresAt: row.expiresAt,
|
||||
expired: row.expiresAt !== null && Date.parse(row.expiresAt) <= now.getTime(),
|
||||
};
|
||||
}
|
||||
|
||||
export async function listApiTokens(userId: number) {
|
||||
return db.select().from(apiTokens).where(eq(apiTokens.userId, userId)).orderBy(apiTokens.createdAt);
|
||||
}
|
||||
|
||||
/** Removes one of a person's own tokens. Returns whether there was one. */
|
||||
export async function removeApiToken(userId: number, id: number): Promise<ApiTokenRow | null> {
|
||||
const [row] = await db.select().from(apiTokens).where(and(eq(apiTokens.id, id), eq(apiTokens.userId, userId))).limit(1);
|
||||
if (!row) return null;
|
||||
await db.delete(apiTokens).where(eq(apiTokens.id, id));
|
||||
return row;
|
||||
}
|
||||
@@ -39,6 +39,7 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
|
||||
push: "settings",
|
||||
share_link: "settings",
|
||||
metrics_token: "settings",
|
||||
api_token: "settings",
|
||||
webhook: "settings",
|
||||
integration: "integrations",
|
||||
dns: "integrations",
|
||||
|
||||
@@ -14,7 +14,8 @@ import tls from "node:tls";
|
||||
import type { TLSSocket } from "node:tls";
|
||||
import { and, asc, eq, gte, inArray, lt, sql } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, endpointDaily, endpointResults } from "../db/schema.js";
|
||||
import { endpointChecks, endpointDaily, endpointHourly, endpointResults } from "../db/schema.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
import { noteRenewal } from "./renewals.js";
|
||||
import { resolveTarget } from "../utils/networkGuard.js";
|
||||
|
||||
@@ -246,6 +247,7 @@ export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c:
|
||||
.where(eq(endpointChecks.id, c.id));
|
||||
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
|
||||
await addToDaily(c.id, iso, result);
|
||||
await addToHourly(c.id, iso, result);
|
||||
// A certificate that is now good for much longer than it was has been renewed.
|
||||
if (result.certExpiresAt) await noteRenewal("certificate", c.name, c.certExpiresAt, result.certExpiresAt, now);
|
||||
return result;
|
||||
@@ -272,6 +274,45 @@ async function addToDaily(checkId: number, atIso: string, result: ProbeResult):
|
||||
}
|
||||
}
|
||||
|
||||
/** How long the hourly figures are kept: between the individual results (two weeks) and the daily figures (years). */
|
||||
export const KEEP_HOURLY_DAYS = 120;
|
||||
|
||||
/** Adds one result to its hour, so an answer-time chart can reach back beyond two weeks with a point an hour rather than one for every check. */
|
||||
async function addToHourly(checkId: number, atIso: string, result: ProbeResult): Promise<void> {
|
||||
const timed = result.ok && result.ms !== null;
|
||||
const ok = result.ok ? 1 : 0;
|
||||
const ms = timed ? (result.ms as number) : 0;
|
||||
const hour = `${atIso.slice(0, 13)}:00:00.000Z`;
|
||||
try {
|
||||
await db
|
||||
.insert(endpointHourly)
|
||||
.values({ checkId, hour, total: 1, ok, msSum: ms, msCount: timed ? 1 : 0, msMax: ms })
|
||||
.onConflictDoUpdate({
|
||||
target: [endpointHourly.checkId, endpointHourly.hour],
|
||||
set: { total: sql`${endpointHourly.total} + 1`, ok: sql`${endpointHourly.ok} + ${ok}`, msSum: sql`${endpointHourly.msSum} + ${ms}`, msCount: sql`${endpointHourly.msCount} + ${timed ? 1 : 0}`, msMax: sql`max(${endpointHourly.msMax}, ${ms})` },
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("[endpoints] couldn't add to the hourly figures:", err);
|
||||
}
|
||||
}
|
||||
|
||||
const HOURLY_BACKFILL_FLAG = "endpointHourlyBackfilled";
|
||||
|
||||
/** The first time after the hourly figures were added, builds them from the results still kept (two weeks). Done once. */
|
||||
export async function backfillEndpointHourly(): Promise<boolean> {
|
||||
if (await getInternalFlag(HOURLY_BACKFILL_FLAG)) return false;
|
||||
await db.run(sql`
|
||||
INSERT INTO endpoint_hourly (check_id, hour, total, ok, ms_sum, ms_count, ms_max)
|
||||
SELECT check_id, substr(at, 1, 13) || ':00:00.000Z', count(*), sum(ok), sum(CASE WHEN ok = 1 AND ms IS NOT NULL THEN ms ELSE 0 END),
|
||||
sum(CASE WHEN ok = 1 AND ms IS NOT NULL THEN 1 ELSE 0 END), coalesce(max(CASE WHEN ok = 1 AND ms IS NOT NULL THEN ms END), 0)
|
||||
FROM endpoint_results
|
||||
GROUP BY check_id, substr(at, 1, 13)
|
||||
ON CONFLICT (check_id, hour) DO UPDATE SET total = excluded.total, ok = excluded.ok, ms_sum = excluded.ms_sum, ms_count = excluded.ms_count, ms_max = excluded.ms_max
|
||||
`);
|
||||
await setInternalFlag(HOURLY_BACKFILL_FLAG, new Date().toISOString());
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
|
||||
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
|
||||
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
|
||||
@@ -290,6 +331,111 @@ export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRo
|
||||
export async function pruneResults(now: Date = new Date()): Promise<void> {
|
||||
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
|
||||
await db.delete(endpointDaily).where(lt(endpointDaily.day, new Date(now.getTime() - KEEP_DAILY_DAYS * 86_400_000).toISOString().slice(0, 10)));
|
||||
await db.delete(endpointHourly).where(lt(endpointHourly.hour, new Date(now.getTime() - KEEP_HOURLY_DAYS * 86_400_000).toISOString()));
|
||||
}
|
||||
|
||||
// ─── How a check has done over a long time ─────────────────────────────────
|
||||
|
||||
/** A row of figures for some stretch of time (one result, one hour or one day), with the moment it starts at as epoch ms. */
|
||||
export interface HistoryRow {
|
||||
t: number;
|
||||
total: number;
|
||||
ok: number;
|
||||
msSum: number;
|
||||
msCount: number;
|
||||
msMax: number;
|
||||
}
|
||||
|
||||
export interface HistoryPoint {
|
||||
/** ISO. */
|
||||
t: string;
|
||||
/** Share of the checks in this stretch that answered, 0–100. */
|
||||
availability: number | null;
|
||||
/** Average and longest answer time of the good, timed answers; null when there were none. */
|
||||
avgMs: number | null;
|
||||
maxMs: number | null;
|
||||
/** How many checks it stands for. */
|
||||
checks: number;
|
||||
}
|
||||
|
||||
/** Rows into at most `maxPoints` equal slices of [from, to]: figures are added up per slice, so a slice is as true as the checks in it. */
|
||||
export function bucketHistory(rows: HistoryRow[], from: number, to: number, maxPoints: number): HistoryPoint[] {
|
||||
if (rows.length === 0 || to <= from) return [];
|
||||
const size = Math.max((to - from) / maxPoints, 1);
|
||||
const buckets = new Map<number, { total: number; ok: number; msSum: number; msCount: number; msMax: number; tSum: number; rows: number }>();
|
||||
for (const r of rows) {
|
||||
if (r.t < from || r.t > to || r.total <= 0) continue;
|
||||
const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size));
|
||||
const b = buckets.get(key) ?? { total: 0, ok: 0, msSum: 0, msCount: 0, msMax: 0, tSum: 0, rows: 0 };
|
||||
b.total += r.total;
|
||||
b.ok += r.ok;
|
||||
b.msSum += r.msSum;
|
||||
b.msCount += r.msCount;
|
||||
b.msMax = Math.max(b.msMax, r.msMax);
|
||||
b.tSum += r.t;
|
||||
b.rows += 1;
|
||||
buckets.set(key, b);
|
||||
}
|
||||
return [...buckets.entries()]
|
||||
.sort((a, b) => a[0] - b[0])
|
||||
.map(([, b]) => ({
|
||||
t: new Date(b.tSum / b.rows).toISOString(),
|
||||
availability: Math.round((b.ok / b.total) * 1000) / 10,
|
||||
avgMs: b.msCount > 0 ? Math.round(b.msSum / b.msCount) : null,
|
||||
maxMs: b.msCount > 0 ? b.msMax : null,
|
||||
checks: b.total,
|
||||
}));
|
||||
}
|
||||
|
||||
export interface EndpointHistory {
|
||||
days: number;
|
||||
/** Which figures it was drawn from: the results themselves (a day or two), the hourly figures, or the daily ones. */
|
||||
source: "results" | "hourly" | "daily";
|
||||
points: HistoryPoint[];
|
||||
/** Over the whole stretch. */
|
||||
availability: number | null;
|
||||
avgMs: number | null;
|
||||
checks: number;
|
||||
}
|
||||
|
||||
/** How a check did over the last `days` (up to a year): availability and answer times, from the finest figures that reach back that far. */
|
||||
export async function getEndpointHistory(checkId: number, days: number, now: Date = new Date()): Promise<EndpointHistory> {
|
||||
days = Math.min(365, Math.max(1, Math.round(days)));
|
||||
const to = now.getTime();
|
||||
const from = to - days * 86_400_000;
|
||||
let rows: HistoryRow[];
|
||||
let source: EndpointHistory["source"];
|
||||
if (days <= 2) {
|
||||
source = "results";
|
||||
const found = await db
|
||||
.select({ at: endpointResults.at, ok: endpointResults.ok, ms: endpointResults.ms })
|
||||
.from(endpointResults)
|
||||
.where(and(eq(endpointResults.checkId, checkId), gte(endpointResults.at, new Date(from).toISOString())));
|
||||
rows = found.map((r) => {
|
||||
const timed = r.ok && r.ms !== null;
|
||||
return { t: Date.parse(r.at), total: 1, ok: r.ok ? 1 : 0, msSum: timed ? (r.ms as number) : 0, msCount: timed ? 1 : 0, msMax: timed ? (r.ms as number) : 0 };
|
||||
});
|
||||
} else if (days <= KEEP_HOURLY_DAYS) {
|
||||
source = "hourly";
|
||||
const found = await db.select().from(endpointHourly).where(and(eq(endpointHourly.checkId, checkId), gte(endpointHourly.hour, new Date(from).toISOString())));
|
||||
rows = found.map((r) => ({ t: Date.parse(r.hour), total: r.total, ok: r.ok, msSum: r.msSum, msCount: r.msCount, msMax: r.msMax }));
|
||||
} else {
|
||||
source = "daily";
|
||||
const found = await db.select().from(endpointDaily).where(and(eq(endpointDaily.checkId, checkId), gte(endpointDaily.day, new Date(from).toISOString().slice(0, 10))));
|
||||
// A day has no record of its slowest answer; its average stands in for it.
|
||||
rows = found.map((r) => ({ t: Date.parse(`${r.day}T00:00:00.000Z`), total: r.total, ok: r.ok, msSum: r.msSum, msCount: r.msCount, msMax: r.msCount > 0 ? Math.round(r.msSum / r.msCount) : 0 }));
|
||||
}
|
||||
const checks = rows.reduce((n, r) => n + r.total, 0);
|
||||
const okCount = rows.reduce((n, r) => n + r.ok, 0);
|
||||
const msCount = rows.reduce((n, r) => n + r.msCount, 0);
|
||||
return {
|
||||
days,
|
||||
source,
|
||||
points: bucketHistory(rows, from, to, days <= 2 ? 96 : days <= KEEP_HOURLY_DAYS ? 200 : 370),
|
||||
availability: checks === 0 ? null : Math.round((okCount / checks) * 1000) / 10,
|
||||
avgMs: msCount === 0 ? null : Math.round(rows.reduce((n, r) => n + r.msSum, 0) / msCount),
|
||||
checks,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── What the history says ─────────────────────────────────────────────────
|
||||
|
||||
@@ -19,7 +19,7 @@ import { getCachedBackupCoverage } from "./backupCoverage.js";
|
||||
import { lastAttempt as lastSelfBackup, listSnapshots } from "./selfBackup.js";
|
||||
import { loadAcks, attentionCounts } from "./alertAcks.js";
|
||||
import { listIncidents } from "./incidents.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
@@ -132,7 +132,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
|
||||
try {
|
||||
for (const s of serverRows) {
|
||||
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), 3650, 101)) {
|
||||
for (const f of disksFillingSoon(await getForecastSeries(s.id, now), 3650, 101)) {
|
||||
out.add("homelab_server_disk_days_to_full", "gauge", "Days until a disk is full at its present rate of filling; only for disks with a steady climb.", f.daysToFull, { server: s.name, mount: f.mount });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* The network equipment read from UniFi controllers: devices with their state, address, uptime and what they are connected through, the networks and the Wi-Fi
|
||||
* names, and which servers are plugged into which device. Shared by the Network devices page and the read API.
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, networkDevices, networkSegments, networkServerLinks, servers } from "../db/schema.js";
|
||||
import { dependentsOf, loadGraph, needsOfNode, nodeKey } from "./dependencies.js";
|
||||
import { activeSubjects } from "./maintenance.js";
|
||||
import { syncStatuses } from "./unifiSync.js";
|
||||
|
||||
export async function buildNetworkOverview() {
|
||||
const [intRows, devices, segments, statuses, graph, silenced, linkRows, serverRows] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "unifi")),
|
||||
db.select().from(networkDevices),
|
||||
db.select().from(networkSegments),
|
||||
syncStatuses(),
|
||||
loadGraph(),
|
||||
activeSubjects(),
|
||||
db.select().from(networkServerLinks),
|
||||
db.select({ id: servers.id, name: servers.name }).from(servers),
|
||||
]);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
const intName = new Map(intRows.map((i) => [i.id, i.name]));
|
||||
const byExternal = new Map(devices.map((d) => [`${d.integrationId}:${d.externalId}`, d]));
|
||||
// The servers plugged into each device, by what the controller says is connected (see unifiSync.matchClientsToServers).
|
||||
const plugged = new Map<number, { serverId: number; serverName: string; port: number | null; wired: boolean }[]>();
|
||||
const serverLinks: { serverId: number; serverName: string; deviceId: number; deviceName: string; port: number | null; wired: boolean }[] = [];
|
||||
for (const l of linkRows) {
|
||||
const device = byExternal.get(`${l.integrationId}:${l.deviceExternalId}`);
|
||||
const name = serverName.get(l.serverId);
|
||||
if (!device || !name) continue;
|
||||
plugged.set(device.id, [...(plugged.get(device.id) ?? []), { serverId: l.serverId, serverName: name, port: l.port, wired: l.wired }]);
|
||||
serverLinks.push({ serverId: l.serverId, serverName: name, deviceId: device.id, deviceName: device.name, port: l.port, wired: l.wired });
|
||||
}
|
||||
return {
|
||||
serverLinks,
|
||||
integrations: intRows.map((i) => ({ id: i.id, name: i.name, enabled: i.enabled, status: statuses[i.id] ?? null })),
|
||||
devices: devices
|
||||
.map((d) => {
|
||||
const key = nodeKey("device", d.id);
|
||||
const uplink = d.uplinkExternalId ? byExternal.get(`${d.integrationId}:${d.uplinkExternalId}`) : undefined;
|
||||
return {
|
||||
id: d.id,
|
||||
integrationId: d.integrationId,
|
||||
integrationName: intName.get(d.integrationId) ?? "UniFi",
|
||||
siteName: d.siteName,
|
||||
name: d.name,
|
||||
model: d.model,
|
||||
kind: d.kind,
|
||||
ip: d.ip,
|
||||
mac: d.mac,
|
||||
state: d.state,
|
||||
uptimeSeconds: d.uptimeSeconds,
|
||||
firmware: d.firmware,
|
||||
firmwareUpdatable: d.firmwareUpdatable,
|
||||
uplink: uplink ? { id: uplink.id, name: uplink.name } : null,
|
||||
offlineSince: d.offlineSince,
|
||||
lastOnlineAt: d.lastOnlineAt,
|
||||
updatedAt: d.updatedAt,
|
||||
// How many things depend on it (directly or through others), and how many it needs: what the dependency map says about it.
|
||||
dependents: dependentsOf(graph.edges, key).length,
|
||||
needs: needsOfNode(graph.edges, key).length,
|
||||
inMaintenance: silenced.has(key),
|
||||
servers: (plugged.get(d.id) ?? []).sort((a, b) => a.serverName.localeCompare(b.serverName)),
|
||||
};
|
||||
})
|
||||
.sort((a, b) => a.name.localeCompare(b.name)),
|
||||
networks: segments.filter((s) => s.kind === "network").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => (a.vlanId ?? 0) - (b.vlanId ?? 0) || a.name.localeCompare(b.name)),
|
||||
ssids: segments.filter((s) => s.kind === "ssid").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* What the read API (/api/v1) says. Each of these is a deliberate, stable shape — not the app's own page payloads, which change with the pages — and only things
|
||||
* any signed-in person can already see: no tokens, no secrets, no passwords, no settings. Dates are ISO 8601 in UTC; sizes are bytes; percentages are 0–100.
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, servers } from "../db/schema.js";
|
||||
import { env } from "../env.js";
|
||||
import { attentionCounts, loadAcks } from "./alertAcks.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { loadStats } from "./endpointChecks.js";
|
||||
import { describeTarget, listActiveWindows, listUpcomingWindows } from "./maintenance.js";
|
||||
import { parseHostHealth } from "./hostHealth.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { parseTags } from "./serverTags.js";
|
||||
import { parseThresholds } from "./serverThresholds.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { serverStatus } from "./topology.js";
|
||||
|
||||
/** An address inside the app as a full address, for a script that has no idea where the app lives. */
|
||||
export function absoluteUrl(path: string | null): string | null {
|
||||
return path === null ? null : `${env.appBaseUrl.replace(/\/+$/, "")}${path.startsWith("/") ? "" : "/"}${path}`;
|
||||
}
|
||||
|
||||
const json = <T>(text: string | null, fallback: T): T => {
|
||||
if (!text) return fallback;
|
||||
try {
|
||||
return JSON.parse(text) as T;
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
};
|
||||
|
||||
type ServerRow = typeof servers.$inferSelect;
|
||||
|
||||
export function serverV1(row: ServerRow, offlineMinutes: number, now: Date) {
|
||||
const status = serverStatus(row, offlineMinutes, now);
|
||||
const disks = json<{ mount: string; sizeBytes: number; usedBytes: number }[]>(row.disks, []);
|
||||
const updates = parsePendingUpdates(row.pendingUpdates);
|
||||
const health = parseHostHealth(row.hostHealth);
|
||||
return {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
hostname: row.hostname,
|
||||
description: row.description,
|
||||
os: row.osType,
|
||||
tags: parseTags(row.tags),
|
||||
ipAddresses: json<string[]>(row.ipAddresses, []),
|
||||
// "online" while its agent has reported within its limit, "offline" after, "unknown" if it never has.
|
||||
status: status === "ok" ? "online" : status === "bad" ? "offline" : "unknown",
|
||||
lastSeenAt: row.lastSeenAt,
|
||||
createdAt: row.createdAt,
|
||||
agentVersion: row.agentVersion ?? null,
|
||||
runsOnServerId: row.parentServerId ?? null,
|
||||
hardware: { manufacturer: row.hwManufacturer ?? null, model: row.hwModel ?? null, serial: row.hwSerial ?? null },
|
||||
cpu: { model: row.cpuModel, cores: row.cpuCores, loadPercent: row.cpuLoadPercent },
|
||||
memory: { totalBytes: row.memTotalBytes, usedBytes: row.memUsedBytes },
|
||||
disks: disks.map((d) => ({ mount: d.mount, sizeBytes: d.sizeBytes, usedBytes: d.usedBytes, usedPercent: d.sizeBytes > 0 ? Math.round((d.usedBytes / d.sizeBytes) * 1000) / 10 : null })),
|
||||
updates: updates ? { manager: updates.manager, count: updates.count, security: updates.security ?? null, rebootRequired: !!updates.rebootRequired, reportedAt: updates.reportedAt ?? null } : null,
|
||||
health: health ? { failedUnits: health.failedUnits?.length ?? 0, reportedAt: health.reportedAt ?? null } : null,
|
||||
thresholds: parseThresholds(row.thresholds),
|
||||
url: absoluteUrl(`/servers/${row.id}`),
|
||||
};
|
||||
}
|
||||
|
||||
export async function serversV1(now: Date = new Date()) {
|
||||
const [rows, settings] = await Promise.all([db.select().from(servers).orderBy(servers.name), getSettings()]);
|
||||
return rows.map((r) => serverV1(r, settings.healthChecks.serverOfflineMinutes, now));
|
||||
}
|
||||
|
||||
export async function oneServerV1(id: number, now: Date = new Date()) {
|
||||
const [row] = await db.select().from(servers).where(eq(servers.id, id)).limit(1);
|
||||
if (!row) return null;
|
||||
return serverV1(row, (await getSettings()).healthChecks.serverOfflineMinutes, now);
|
||||
}
|
||||
|
||||
export async function alertsV1() {
|
||||
const [report, acks] = await Promise.all([getAlerts(false), loadAcks()]);
|
||||
const attention = attentionCounts(report.alerts, acks);
|
||||
return {
|
||||
generatedAt: report.generatedAt,
|
||||
// What is on the page: those nobody has looked at, by severity, and how many somebody has.
|
||||
needingAttention: { critical: attention.critical, warning: attention.warning, acknowledged: attention.acknowledged },
|
||||
counts: report.counts,
|
||||
couldntCheck: report.couldntCheck,
|
||||
alerts: report.alerts.map((a) => {
|
||||
const ack = acks.get(a.id);
|
||||
return {
|
||||
id: a.id,
|
||||
severity: a.severity,
|
||||
category: a.category,
|
||||
source: a.source,
|
||||
message: a.message,
|
||||
url: absoluteUrl(a.link),
|
||||
silenced: a.silenced,
|
||||
flapping: !!a.flapping,
|
||||
target: a.target,
|
||||
causedBy: a.causedBy ?? null,
|
||||
affects: a.affects ?? [],
|
||||
acknowledgement: ack?.active ? { by: ack.ackedBy, at: ack.ackedAt, note: ack.note, until: ack.until, assignee: ack.assignee?.name ?? null } : null,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function inventoryV1() {
|
||||
const [rows, serverRows] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select({ id: servers.id, name: servers.name }).from(servers)]);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
return rows.map((a) => ({
|
||||
id: a.id,
|
||||
name: a.name,
|
||||
kind: a.kind,
|
||||
manufacturer: a.manufacturer,
|
||||
model: a.model,
|
||||
serial: a.serial,
|
||||
location: a.location,
|
||||
rackUnit: a.rackUnit,
|
||||
purchaseDate: a.purchaseDate,
|
||||
purchasePrice: a.purchasePrice,
|
||||
warrantyUntil: a.warrantyUntil,
|
||||
endOfSupport: a.endOfSupport,
|
||||
replacementCost: a.replacementCost,
|
||||
powerWatts: a.powerWatts,
|
||||
notes: a.notes,
|
||||
server: a.serverId !== null && serverName.has(a.serverId) ? { id: a.serverId, name: serverName.get(a.serverId)! } : null,
|
||||
updatedAt: a.updatedAt,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function maintenanceV1(now: Date = new Date()) {
|
||||
const shape = async (w: Awaited<ReturnType<typeof listActiveWindows>>[number]) => {
|
||||
const target = await describeTarget(w.targetType, w.targetId);
|
||||
return target ? { id: w.id, target: { type: w.targetType, id: w.targetId, name: target.name, kind: target.kind }, reason: w.reason, startsAt: w.startedAt, endsAt: w.endsAt, createdBy: w.createdBy } : null;
|
||||
};
|
||||
const active = (await Promise.all((await listActiveWindows(now)).map(shape))).filter((w): w is NonNullable<typeof w> => w !== null);
|
||||
const upcoming = (await Promise.all((await listUpcomingWindows(now)).map(shape))).filter((w): w is NonNullable<typeof w> => w !== null);
|
||||
return { active, upcoming };
|
||||
}
|
||||
|
||||
export async function endpointsV1(now: Date = new Date()) {
|
||||
const rows = await db.select().from(endpointChecks).orderBy(endpointChecks.name);
|
||||
const stats = await loadStats(rows.map((r) => r.id), now);
|
||||
return rows.map((r) => {
|
||||
const st = stats.get(r.id);
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
kind: r.kind,
|
||||
target: r.target,
|
||||
port: r.port,
|
||||
enabled: r.enabled,
|
||||
status: !r.enabled ? "paused" : r.lastOk === null ? "unknown" : r.lastOk ? "up" : "down",
|
||||
lastCheckedAt: r.lastCheckedAt,
|
||||
lastResponseMs: r.lastResponseMs,
|
||||
lastMessage: r.lastOk === false ? r.lastMessage : null,
|
||||
certificateExpiresAt: r.certExpiresAt,
|
||||
uptime24hPercent: st?.uptime24h ?? null,
|
||||
uptime7dPercent: st?.uptime7d ?? null,
|
||||
averageResponseMs24h: st?.avgMs24h ?? null,
|
||||
serverId: r.serverId,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -1,19 +1,30 @@
|
||||
/**
|
||||
* A short history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend
|
||||
* A history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend
|
||||
* (and say how soon a filling disk will run out) instead of only the latest figure.
|
||||
*
|
||||
* One sample is kept per server every ten minutes at most, however often its agent reports, for 35 days. Everything here is a
|
||||
* percentage: CPU load (as the agent reports it), memory in use, and each disk's used share.
|
||||
*
|
||||
* Beyond those 35 days the same figures are kept coarser: each sample also goes into its hour (kept four months) and its day (kept over two years), as a
|
||||
* sum, a count, a lowest and a highest. A chart asks for a number of days and is answered from whichever is fine enough and still there: the samples up to
|
||||
* 35 days, the hours up to 120, the days beyond. A year of a disk filling is then one point a day.
|
||||
*/
|
||||
import { and, desc, eq, gte, lt } from "drizzle-orm";
|
||||
import { and, desc, eq, gte, lt, sql } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { serverMetrics } from "../db/schema.js";
|
||||
import { serverMetrics, serverMetricsRollup } from "../db/schema.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
import type { IncomingSystemInfo } from "./taskSync.js";
|
||||
|
||||
/** Reports closer together than this add no sample. */
|
||||
export const MIN_GAP_MS = 10 * 60_000;
|
||||
export const RETENTION_DAYS = 35;
|
||||
/** How long the hourly and daily figures are kept. */
|
||||
export const HOURLY_KEEP_DAYS = 120;
|
||||
export const DAILY_KEEP_DAYS = 800;
|
||||
/** The longest stretch a chart may ask for. */
|
||||
export const MAX_RANGE_DAYS = 365;
|
||||
const DAY_MS = 24 * 3600_000;
|
||||
const HOUR_MS = 3600_000;
|
||||
|
||||
export interface Sample {
|
||||
metric: string;
|
||||
@@ -39,6 +50,35 @@ export function samplesFromSystem(system: IncomingSystemInfo | null | undefined)
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The start of the hour or day a moment falls in, as ISO. */
|
||||
export const hourStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / HOUR_MS) * HOUR_MS).toISOString();
|
||||
export const dayStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / DAY_MS) * DAY_MS).toISOString();
|
||||
|
||||
/** Adds samples to their hour and day. Never allowed to break recording the sample itself. */
|
||||
async function addToRollups(serverId: number, samples: Sample[], at: Date): Promise<void> {
|
||||
try {
|
||||
for (const grain of ["h", "d"] as const) {
|
||||
const bucket = grain === "h" ? hourStart(at.getTime()) : dayStart(at.getTime());
|
||||
for (const s of samples) {
|
||||
await db
|
||||
.insert(serverMetricsRollup)
|
||||
.values({ serverId, metric: s.metric, grain, at: bucket, sum: s.value, n: 1, min: s.value, max: s.value })
|
||||
.onConflictDoUpdate({
|
||||
target: [serverMetricsRollup.serverId, serverMetricsRollup.metric, serverMetricsRollup.grain, serverMetricsRollup.at],
|
||||
set: {
|
||||
sum: sql`${serverMetricsRollup.sum} + ${s.value}`,
|
||||
n: sql`${serverMetricsRollup.n} + 1`,
|
||||
min: sql`min(${serverMetricsRollup.min}, ${s.value})`,
|
||||
max: sql`max(${serverMetricsRollup.max}, ${s.value})`,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[metrics] couldn't add to the hourly and daily figures:", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Records a report's figures, unless this server was sampled a moment ago. Returns whether anything was stored. */
|
||||
export async function recordServerMetrics(serverId: number, system: IncomingSystemInfo | null | undefined, now: Date = new Date()): Promise<boolean> {
|
||||
const samples = samplesFromSystem(system);
|
||||
@@ -54,7 +94,32 @@ export async function recordServerMetrics(serverId: number, system: IncomingSyst
|
||||
|
||||
const at = now.toISOString();
|
||||
await db.insert(serverMetrics).values(samples.map((s) => ({ serverId, metric: s.metric, value: s.value, recordedAt: at })));
|
||||
await addToRollups(serverId, samples, now);
|
||||
await db.delete(serverMetrics).where(and(eq(serverMetrics.serverId, serverId), lt(serverMetrics.recordedAt, new Date(now.getTime() - RETENTION_DAYS * DAY_MS).toISOString())));
|
||||
await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "h"), lt(serverMetricsRollup.at, new Date(now.getTime() - HOURLY_KEEP_DAYS * DAY_MS).toISOString())));
|
||||
await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "d"), lt(serverMetricsRollup.at, new Date(now.getTime() - DAILY_KEEP_DAYS * DAY_MS).toISOString())));
|
||||
return true;
|
||||
}
|
||||
|
||||
const BACKFILL_FLAG = "metricsRollupBackfilled";
|
||||
|
||||
/**
|
||||
* The first time after the hourly and daily figures were added, builds them from the samples already kept (up to 35 days), so the longer ranges are not empty
|
||||
* for weeks. A bucket is written from what the samples say, replacing what is there, so it is right however far it got before. Done once.
|
||||
*/
|
||||
export async function backfillMetricRollups(): Promise<boolean> {
|
||||
if (await getInternalFlag(BACKFILL_FLAG)) return false;
|
||||
for (const [grain, len] of [["h", 13], ["d", 10]] as const) {
|
||||
const suffix = grain === "h" ? ":00:00.000Z" : "T00:00:00.000Z";
|
||||
await db.run(sql`
|
||||
INSERT INTO server_metrics_rollup (server_id, metric, grain, at, sum, n, min, max)
|
||||
SELECT server_id, metric, ${grain}, substr(recorded_at, 1, ${len}) || ${suffix}, sum(value), count(*), min(value), max(value)
|
||||
FROM server_metrics
|
||||
GROUP BY server_id, metric, substr(recorded_at, 1, ${len})
|
||||
ON CONFLICT (server_id, metric, grain, at) DO UPDATE SET sum = excluded.sum, n = excluded.n, min = excluded.min, max = excluded.max
|
||||
`);
|
||||
}
|
||||
await setInternalFlag(BACKFILL_FLAG, new Date().toISOString());
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -62,6 +127,9 @@ export interface Point {
|
||||
/** ISO time. */
|
||||
t: string;
|
||||
v: number;
|
||||
/** The lowest and highest figure in what this point stands for, when it stands for several (hourly and daily figures). */
|
||||
lo?: number;
|
||||
hi?: number;
|
||||
}
|
||||
|
||||
/** Averages points into at most `maxPoints` equal slices of [from, to], so a month of samples draws as a short line. */
|
||||
@@ -83,6 +151,42 @@ export function bucketPoints(points: { t: number; v: number }[], from: number, t
|
||||
.map(([, b]) => ({ t: new Date(b.tSum / b.n).toISOString(), v: round1(b.sum / b.n) }));
|
||||
}
|
||||
|
||||
/** A row of the hourly or daily figures, with the moment it starts at as epoch ms. */
|
||||
export interface RollupRow {
|
||||
t: number;
|
||||
sum: number;
|
||||
n: number;
|
||||
min: number;
|
||||
max: number;
|
||||
}
|
||||
|
||||
/** The average of what each row stands for. */
|
||||
export function rollupAverages(rows: RollupRow[]): { t: number; v: number }[] {
|
||||
return rows.filter((r) => r.n > 0).map((r) => ({ t: r.t, v: r.sum / r.n }));
|
||||
}
|
||||
|
||||
/** Rows into at most `maxPoints` slices of [from, to]: the average of the samples in each (not of the rows), with the lowest and highest. */
|
||||
export function bucketRollups(rows: RollupRow[], from: number, to: number, maxPoints: number): Point[] {
|
||||
if (rows.length === 0 || to <= from) return [];
|
||||
const size = Math.max((to - from) / maxPoints, 1);
|
||||
const buckets = new Map<number, { sum: number; n: number; lo: number; hi: number; tSum: number; rows: number }>();
|
||||
for (const r of rows) {
|
||||
if (r.t < from || r.t > to || r.n <= 0) continue;
|
||||
const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size));
|
||||
const b = buckets.get(key) ?? { sum: 0, n: 0, lo: Infinity, hi: -Infinity, tSum: 0, rows: 0 };
|
||||
b.sum += r.sum;
|
||||
b.n += r.n;
|
||||
b.lo = Math.min(b.lo, r.min);
|
||||
b.hi = Math.max(b.hi, r.max);
|
||||
b.tSum += r.t;
|
||||
b.rows += 1;
|
||||
buckets.set(key, b);
|
||||
}
|
||||
return [...buckets.entries()]
|
||||
.sort((a, b) => a[0] - b[0])
|
||||
.map(([, b]) => ({ t: new Date(b.tSum / b.rows).toISOString(), v: round1(b.sum / b.n), lo: round1(b.lo), hi: round1(b.hi) }));
|
||||
}
|
||||
|
||||
export interface Trend {
|
||||
/** Change in the percentage per day, from a straight line through the points. */
|
||||
perDay: number;
|
||||
@@ -175,35 +279,86 @@ function describe(metric: string): { kind: MetricSeries["kind"]; label: string }
|
||||
|
||||
const ORDER = { cpu: 0, mem: 1, disk: 2 } as const;
|
||||
|
||||
/** Every series recorded for a server over the last `days`, ready to draw. */
|
||||
/** Which of the kept figures answers a stretch of `days`: the samples while they reach back that far, then the hours, then the days. */
|
||||
export function sourceFor(days: number): "samples" | "hourly" | "daily" {
|
||||
return days <= RETENTION_DAYS ? "samples" : days <= HOURLY_KEEP_DAYS ? "hourly" : "daily";
|
||||
}
|
||||
|
||||
/** How many points a chart over `days` is drawn from at most. */
|
||||
const pointsFor = (days: number) => (days <= RETENTION_DAYS ? 96 : days <= HOURLY_KEEP_DAYS ? 200 : MAX_RANGE_DAYS + 5);
|
||||
|
||||
/** The latest sample of each figure of a server, whatever range is asked for. */
|
||||
async function latestSamples(serverId: number): Promise<Map<string, number>> {
|
||||
const rows = await db.all<{ metric: string; value: number }>(
|
||||
sql`SELECT metric, value FROM server_metrics WHERE id IN (SELECT max(id) FROM server_metrics WHERE server_id = ${serverId} GROUP BY metric)`,
|
||||
);
|
||||
return new Map(rows.map((r) => [r.metric, r.value]));
|
||||
}
|
||||
|
||||
/** Every series recorded for a server over the last `days` (up to a year), ready to draw. */
|
||||
export async function getServerMetrics(serverId: number, days: number, now: Date = new Date()): Promise<MetricSeries[]> {
|
||||
days = Math.min(MAX_RANGE_DAYS, Math.max(1, days));
|
||||
const to = now.getTime();
|
||||
const from = to - days * DAY_MS;
|
||||
const rows = await db
|
||||
.select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt })
|
||||
.from(serverMetrics)
|
||||
.where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString())))
|
||||
.orderBy(serverMetrics.recordedAt);
|
||||
const source = sourceFor(days);
|
||||
|
||||
const byMetric = new Map<string, { t: number; v: number }[]>();
|
||||
for (const r of rows) {
|
||||
const list = byMetric.get(r.metric) ?? [];
|
||||
list.push({ t: Date.parse(r.at), v: r.value });
|
||||
byMetric.set(r.metric, list);
|
||||
// The figures, per metric, as { t, v } for a trend and as points for the chart.
|
||||
const raw = new Map<string, { t: number; v: number }[]>();
|
||||
const points = new Map<string, Point[]>();
|
||||
if (source === "samples") {
|
||||
const rows = await db
|
||||
.select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt })
|
||||
.from(serverMetrics)
|
||||
.where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString())))
|
||||
.orderBy(serverMetrics.recordedAt);
|
||||
for (const r of rows) raw.set(r.metric, [...(raw.get(r.metric) ?? []), { t: Date.parse(r.at), v: r.value }]);
|
||||
for (const [metric, list] of raw) points.set(metric, bucketPoints(list, from, to, pointsFor(days)));
|
||||
} else {
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(serverMetricsRollup)
|
||||
.where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, source === "hourly" ? "h" : "d"), gte(serverMetricsRollup.at, new Date(from).toISOString())))
|
||||
.orderBy(serverMetricsRollup.at);
|
||||
const byMetric = new Map<string, RollupRow[]>();
|
||||
for (const r of rows) byMetric.set(r.metric, [...(byMetric.get(r.metric) ?? []), { t: Date.parse(r.at), sum: r.sum, n: r.n, min: r.min, max: r.max }]);
|
||||
for (const [metric, list] of byMetric) {
|
||||
raw.set(metric, rollupAverages(list));
|
||||
points.set(metric, bucketRollups(list, from, to, pointsFor(days)));
|
||||
}
|
||||
}
|
||||
const latest = source === "samples" ? null : await latestSamples(serverId);
|
||||
|
||||
return [...byMetric.entries()]
|
||||
.map(([metric, raw]) => {
|
||||
return [...raw.entries()]
|
||||
.map(([metric, list]) => {
|
||||
const { kind, label } = describe(metric);
|
||||
const newest = latest?.get(metric) ?? list[list.length - 1]?.v;
|
||||
return {
|
||||
metric,
|
||||
kind,
|
||||
label,
|
||||
points: bucketPoints(raw, from, to, 96),
|
||||
latest: raw[raw.length - 1]?.v ?? null,
|
||||
points: points.get(metric) ?? [],
|
||||
latest: newest === undefined ? null : source === "samples" ? newest : round1(newest),
|
||||
// A disk filling is the thing worth forecasting; load and memory go up and down by the hour.
|
||||
trend: kind === "disk" ? trendOf(raw) : null,
|
||||
trend: kind === "disk" ? trendOf(list) : null,
|
||||
};
|
||||
})
|
||||
.sort((a, b) => ORDER[a.kind] - ORDER[b.kind] || a.label.localeCompare(b.label));
|
||||
}
|
||||
|
||||
/** How far back the longer look at a disk goes when the last two weeks say it is flat. */
|
||||
export const LONG_FORECAST_DAYS = 90;
|
||||
|
||||
/**
|
||||
* The series to forecast from: the last two weeks, as before — except that a disk those two weeks call flat is looked at again over the last
|
||||
* ninety days, so a slow, steady climb (a point a week) is seen too. The longer trend replaces the short one only if it is really rising.
|
||||
*/
|
||||
export async function getForecastSeries(serverId: number, now: Date = new Date()): Promise<MetricSeries[]> {
|
||||
const short = await getServerMetrics(serverId, 14, now);
|
||||
if (!short.some((s) => s.kind === "disk" && (!s.trend || s.trend.daysToFull === null))) return short;
|
||||
const long = await getServerMetrics(serverId, LONG_FORECAST_DAYS, now);
|
||||
return short.map((s) => {
|
||||
if (s.kind !== "disk" || (s.trend && s.trend.daysToFull !== null)) return s;
|
||||
const longer = long.find((l) => l.metric === s.metric);
|
||||
return longer?.trend && longer.trend.daysToFull !== null ? { ...s, trend: longer.trend } : s;
|
||||
});
|
||||
}
|
||||
@@ -16,7 +16,7 @@ import { describeTarget, listUpcomingWindows } from "./maintenance.js";
|
||||
import { notify, notifyNow } from "./notify.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
@@ -163,7 +163,7 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
const own = parseThresholds(s.thresholds);
|
||||
const forecastDays = effectiveThresholds(settings.healthChecks, own).diskForecastDays;
|
||||
if (!(forecastDays > 0)) continue;
|
||||
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
|
||||
for (const f of disksFillingSoon(await getForecastSeries(s.id, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
|
||||
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user