A year of history, one chart, and a read API with personal tokens
History: server load, memory and disks are also kept as hourly (four months) and daily (over two years) averages with lowest and highest; endpoint answers as hourly (four months) beside the daily figures. Charts ask for 24 hours to a year and are drawn from whichever is fine enough. A disk that two weeks call flat is looked at again over ninety days for the forecast. Built once from what is kept. One chart: TimeChart replaces the sparkline and the multi-line chart: hover and arrow-key readout, a lowest/highest band, gaps shown as gaps, the same ranges everywhere and the choice remembered. Used for server trends, comparing servers, a check's availability and answer times (Endpoints > History) and a server's availability. Read API: /api/v1 (servers, metrics, alerts, incidents, inventory, network, endpoints and their history, maintenance) with personal read-only tokens made under the user menu: shown once, hashed, 30/90/365 days or none, ten at most, removed with their owner. GET only. Documented in API.md. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5e26e77340
commit
68b41b5586
47 files changed
+12360
-247
No files matched your search
@@ -12,7 +12,7 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, heartbeats, integrations, networkDevices, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
import { recordFromReport } from "./incidents.js";
|
||||
@@ -499,7 +499,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const own = parseThresholds(s.thresholds);
|
||||
const forecastDays = effectiveThresholds(healthChecks, own).diskForecastDays;
|
||||
if (!(forecastDays > 0)) continue;
|
||||
const series = await getServerMetrics(s.id, 14);
|
||||
const series = await getForecastSeries(s.id);
|
||||
for (const f of disksFillingSoon(series, forecastDays, (mount) => diskLimit(own, mount, healthChecks.diskUsagePercent))) {
|
||||
push({
|
||||
category: "disk",
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* Personal API tokens: read-only access to /api/v1 for scripts, Home Assistant and the like. Each belongs to the person who made it — they see and remove their own —
|
||||
* and goes when they do. Like an agent token or a share link, the secret is shown once, when it is made, and only its SHA-256 is kept.
|
||||
*/
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { apiTokens, users } from "../db/schema.js";
|
||||
|
||||
export type ApiTokenRow = typeof apiTokens.$inferSelect;
|
||||
export type ApiTokenUser = typeof users.$inferSelect;
|
||||
|
||||
const hash = (token: string) => createHash("sha256").update(token).digest("hex");
|
||||
|
||||
/** Most tokens one person may have: one for each thing that reads, and a spare. */
|
||||
export const MAX_TOKENS_PER_PERSON = 10;
|
||||
/** How long a token may be asked to last, in days (or never). */
|
||||
export const EXPIRY_CHOICES = [30, 90, 365] as const;
|
||||
|
||||
/** Use is not recorded more than once in this time, so a script that reads every few seconds doesn't write the database each time. */
|
||||
const USED_WRITE_GAP_MS = 60_000;
|
||||
|
||||
export class TokenLimitError extends Error {}
|
||||
|
||||
export async function createApiToken(params: { userId: number; label: string; expiresInDays: number | null }, now: Date = new Date()): Promise<{ row: ApiTokenRow; token: string }> {
|
||||
const existing = await db.select({ id: apiTokens.id }).from(apiTokens).where(eq(apiTokens.userId, params.userId));
|
||||
if (existing.length >= MAX_TOKENS_PER_PERSON) throw new TokenLimitError(`There are already ${MAX_TOKENS_PER_PERSON} tokens. Remove one you don't use first.`);
|
||||
const token = `hlm_p_${randomBytes(24).toString("base64url")}`;
|
||||
const expiresAt = params.expiresInDays === null ? null : new Date(now.getTime() + params.expiresInDays * 86_400_000).toISOString();
|
||||
const [row] = await db.insert(apiTokens).values({ userId: params.userId, label: params.label, tokenHash: hash(token), tokenPrefix: token.slice(0, 12), expiresAt }).returning();
|
||||
return { row, token };
|
||||
}
|
||||
|
||||
/** The token a secret belongs to and whose it is, if it is one that still works; notes that it was used. */
|
||||
export async function resolveApiToken(secret: string, now: Date = new Date()): Promise<{ token: ApiTokenRow; user: ApiTokenUser } | null> {
|
||||
if (typeof secret !== "string" || secret.length < 20 || secret.length > 128) return null;
|
||||
const [found] = await db
|
||||
.select({ token: apiTokens, user: users })
|
||||
.from(apiTokens)
|
||||
.innerJoin(users, eq(users.id, apiTokens.userId))
|
||||
.where(eq(apiTokens.tokenHash, hash(secret)))
|
||||
.limit(1);
|
||||
if (!found) return null;
|
||||
if (found.token.expiresAt && Date.parse(found.token.expiresAt) <= now.getTime()) return null;
|
||||
if (!found.token.lastUsedAt || now.getTime() - Date.parse(found.token.lastUsedAt) >= USED_WRITE_GAP_MS) {
|
||||
await db.update(apiTokens).set({ lastUsedAt: now.toISOString() }).where(eq(apiTokens.id, found.token.id));
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/** A token as its owner's list shows it: everything but the hash. */
|
||||
export function publicApiToken(row: ApiTokenRow, now: Date = new Date()) {
|
||||
return {
|
||||
id: row.id,
|
||||
label: row.label,
|
||||
tokenPrefix: row.tokenPrefix,
|
||||
createdAt: row.createdAt,
|
||||
lastUsedAt: row.lastUsedAt,
|
||||
expiresAt: row.expiresAt,
|
||||
expired: row.expiresAt !== null && Date.parse(row.expiresAt) <= now.getTime(),
|
||||
};
|
||||
}
|
||||
|
||||
export async function listApiTokens(userId: number) {
|
||||
return db.select().from(apiTokens).where(eq(apiTokens.userId, userId)).orderBy(apiTokens.createdAt);
|
||||
}
|
||||
|
||||
/** Removes one of a person's own tokens. Returns whether there was one. */
|
||||
export async function removeApiToken(userId: number, id: number): Promise<ApiTokenRow | null> {
|
||||
const [row] = await db.select().from(apiTokens).where(and(eq(apiTokens.id, id), eq(apiTokens.userId, userId))).limit(1);
|
||||
if (!row) return null;
|
||||
await db.delete(apiTokens).where(eq(apiTokens.id, id));
|
||||
return row;
|
||||
}
|
||||
@@ -39,6 +39,7 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
|
||||
push: "settings",
|
||||
share_link: "settings",
|
||||
metrics_token: "settings",
|
||||
api_token: "settings",
|
||||
webhook: "settings",
|
||||
integration: "integrations",
|
||||
dns: "integrations",
|
||||
|
||||
@@ -14,7 +14,8 @@ import tls from "node:tls";
|
||||
import type { TLSSocket } from "node:tls";
|
||||
import { and, asc, eq, gte, inArray, lt, sql } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, endpointDaily, endpointResults } from "../db/schema.js";
|
||||
import { endpointChecks, endpointDaily, endpointHourly, endpointResults } from "../db/schema.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
import { noteRenewal } from "./renewals.js";
|
||||
import { resolveTarget } from "../utils/networkGuard.js";
|
||||
|
||||
@@ -246,6 +247,7 @@ export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c:
|
||||
.where(eq(endpointChecks.id, c.id));
|
||||
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
|
||||
await addToDaily(c.id, iso, result);
|
||||
await addToHourly(c.id, iso, result);
|
||||
// A certificate that is now good for much longer than it was has been renewed.
|
||||
if (result.certExpiresAt) await noteRenewal("certificate", c.name, c.certExpiresAt, result.certExpiresAt, now);
|
||||
return result;
|
||||
@@ -272,6 +274,45 @@ async function addToDaily(checkId: number, atIso: string, result: ProbeResult):
|
||||
}
|
||||
}
|
||||
|
||||
/** How long the hourly figures are kept: between the individual results (two weeks) and the daily figures (years). */
|
||||
export const KEEP_HOURLY_DAYS = 120;
|
||||
|
||||
/** Adds one result to its hour, so an answer-time chart can reach back beyond two weeks with a point an hour rather than one for every check. */
|
||||
async function addToHourly(checkId: number, atIso: string, result: ProbeResult): Promise<void> {
|
||||
const timed = result.ok && result.ms !== null;
|
||||
const ok = result.ok ? 1 : 0;
|
||||
const ms = timed ? (result.ms as number) : 0;
|
||||
const hour = `${atIso.slice(0, 13)}:00:00.000Z`;
|
||||
try {
|
||||
await db
|
||||
.insert(endpointHourly)
|
||||
.values({ checkId, hour, total: 1, ok, msSum: ms, msCount: timed ? 1 : 0, msMax: ms })
|
||||
.onConflictDoUpdate({
|
||||
target: [endpointHourly.checkId, endpointHourly.hour],
|
||||
set: { total: sql`${endpointHourly.total} + 1`, ok: sql`${endpointHourly.ok} + ${ok}`, msSum: sql`${endpointHourly.msSum} + ${ms}`, msCount: sql`${endpointHourly.msCount} + ${timed ? 1 : 0}`, msMax: sql`max(${endpointHourly.msMax}, ${ms})` },
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("[endpoints] couldn't add to the hourly figures:", err);
|
||||
}
|
||||
}
|
||||
|
||||
const HOURLY_BACKFILL_FLAG = "endpointHourlyBackfilled";
|
||||
|
||||
/** The first time after the hourly figures were added, builds them from the results still kept (two weeks). Done once. */
|
||||
export async function backfillEndpointHourly(): Promise<boolean> {
|
||||
if (await getInternalFlag(HOURLY_BACKFILL_FLAG)) return false;
|
||||
await db.run(sql`
|
||||
INSERT INTO endpoint_hourly (check_id, hour, total, ok, ms_sum, ms_count, ms_max)
|
||||
SELECT check_id, substr(at, 1, 13) || ':00:00.000Z', count(*), sum(ok), sum(CASE WHEN ok = 1 AND ms IS NOT NULL THEN ms ELSE 0 END),
|
||||
sum(CASE WHEN ok = 1 AND ms IS NOT NULL THEN 1 ELSE 0 END), coalesce(max(CASE WHEN ok = 1 AND ms IS NOT NULL THEN ms END), 0)
|
||||
FROM endpoint_results
|
||||
GROUP BY check_id, substr(at, 1, 13)
|
||||
ON CONFLICT (check_id, hour) DO UPDATE SET total = excluded.total, ok = excluded.ok, ms_sum = excluded.ms_sum, ms_count = excluded.ms_count, ms_max = excluded.ms_max
|
||||
`);
|
||||
await setInternalFlag(HOURLY_BACKFILL_FLAG, new Date().toISOString());
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
|
||||
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
|
||||
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
|
||||
@@ -290,6 +331,111 @@ export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRo
|
||||
export async function pruneResults(now: Date = new Date()): Promise<void> {
|
||||
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
|
||||
await db.delete(endpointDaily).where(lt(endpointDaily.day, new Date(now.getTime() - KEEP_DAILY_DAYS * 86_400_000).toISOString().slice(0, 10)));
|
||||
await db.delete(endpointHourly).where(lt(endpointHourly.hour, new Date(now.getTime() - KEEP_HOURLY_DAYS * 86_400_000).toISOString()));
|
||||
}
|
||||
|
||||
// ─── How a check has done over a long time ─────────────────────────────────
|
||||
|
||||
/** A row of figures for some stretch of time (one result, one hour or one day), with the moment it starts at as epoch ms. */
|
||||
export interface HistoryRow {
|
||||
t: number;
|
||||
total: number;
|
||||
ok: number;
|
||||
msSum: number;
|
||||
msCount: number;
|
||||
msMax: number;
|
||||
}
|
||||
|
||||
export interface HistoryPoint {
|
||||
/** ISO. */
|
||||
t: string;
|
||||
/** Share of the checks in this stretch that answered, 0–100. */
|
||||
availability: number | null;
|
||||
/** Average and longest answer time of the good, timed answers; null when there were none. */
|
||||
avgMs: number | null;
|
||||
maxMs: number | null;
|
||||
/** How many checks it stands for. */
|
||||
checks: number;
|
||||
}
|
||||
|
||||
/** Rows into at most `maxPoints` equal slices of [from, to]: figures are added up per slice, so a slice is as true as the checks in it. */
|
||||
export function bucketHistory(rows: HistoryRow[], from: number, to: number, maxPoints: number): HistoryPoint[] {
|
||||
if (rows.length === 0 || to <= from) return [];
|
||||
const size = Math.max((to - from) / maxPoints, 1);
|
||||
const buckets = new Map<number, { total: number; ok: number; msSum: number; msCount: number; msMax: number; tSum: number; rows: number }>();
|
||||
for (const r of rows) {
|
||||
if (r.t < from || r.t > to || r.total <= 0) continue;
|
||||
const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size));
|
||||
const b = buckets.get(key) ?? { total: 0, ok: 0, msSum: 0, msCount: 0, msMax: 0, tSum: 0, rows: 0 };
|
||||
b.total += r.total;
|
||||
b.ok += r.ok;
|
||||
b.msSum += r.msSum;
|
||||
b.msCount += r.msCount;
|
||||
b.msMax = Math.max(b.msMax, r.msMax);
|
||||
b.tSum += r.t;
|
||||
b.rows += 1;
|
||||
buckets.set(key, b);
|
||||
}
|
||||
return [...buckets.entries()]
|
||||
.sort((a, b) => a[0] - b[0])
|
||||
.map(([, b]) => ({
|
||||
t: new Date(b.tSum / b.rows).toISOString(),
|
||||
availability: Math.round((b.ok / b.total) * 1000) / 10,
|
||||
avgMs: b.msCount > 0 ? Math.round(b.msSum / b.msCount) : null,
|
||||
maxMs: b.msCount > 0 ? b.msMax : null,
|
||||
checks: b.total,
|
||||
}));
|
||||
}
|
||||
|
||||
export interface EndpointHistory {
|
||||
days: number;
|
||||
/** Which figures it was drawn from: the results themselves (a day or two), the hourly figures, or the daily ones. */
|
||||
source: "results" | "hourly" | "daily";
|
||||
points: HistoryPoint[];
|
||||
/** Over the whole stretch. */
|
||||
availability: number | null;
|
||||
avgMs: number | null;
|
||||
checks: number;
|
||||
}
|
||||
|
||||
/** How a check did over the last `days` (up to a year): availability and answer times, from the finest figures that reach back that far. */
|
||||
export async function getEndpointHistory(checkId: number, days: number, now: Date = new Date()): Promise<EndpointHistory> {
|
||||
days = Math.min(365, Math.max(1, Math.round(days)));
|
||||
const to = now.getTime();
|
||||
const from = to - days * 86_400_000;
|
||||
let rows: HistoryRow[];
|
||||
let source: EndpointHistory["source"];
|
||||
if (days <= 2) {
|
||||
source = "results";
|
||||
const found = await db
|
||||
.select({ at: endpointResults.at, ok: endpointResults.ok, ms: endpointResults.ms })
|
||||
.from(endpointResults)
|
||||
.where(and(eq(endpointResults.checkId, checkId), gte(endpointResults.at, new Date(from).toISOString())));
|
||||
rows = found.map((r) => {
|
||||
const timed = r.ok && r.ms !== null;
|
||||
return { t: Date.parse(r.at), total: 1, ok: r.ok ? 1 : 0, msSum: timed ? (r.ms as number) : 0, msCount: timed ? 1 : 0, msMax: timed ? (r.ms as number) : 0 };
|
||||
});
|
||||
} else if (days <= KEEP_HOURLY_DAYS) {
|
||||
source = "hourly";
|
||||
const found = await db.select().from(endpointHourly).where(and(eq(endpointHourly.checkId, checkId), gte(endpointHourly.hour, new Date(from).toISOString())));
|
||||
rows = found.map((r) => ({ t: Date.parse(r.hour), total: r.total, ok: r.ok, msSum: r.msSum, msCount: r.msCount, msMax: r.msMax }));
|
||||
} else {
|
||||
source = "daily";
|
||||
const found = await db.select().from(endpointDaily).where(and(eq(endpointDaily.checkId, checkId), gte(endpointDaily.day, new Date(from).toISOString().slice(0, 10))));
|
||||
// A day has no record of its slowest answer; its average stands in for it.
|
||||
rows = found.map((r) => ({ t: Date.parse(`${r.day}T00:00:00.000Z`), total: r.total, ok: r.ok, msSum: r.msSum, msCount: r.msCount, msMax: r.msCount > 0 ? Math.round(r.msSum / r.msCount) : 0 }));
|
||||
}
|
||||
const checks = rows.reduce((n, r) => n + r.total, 0);
|
||||
const okCount = rows.reduce((n, r) => n + r.ok, 0);
|
||||
const msCount = rows.reduce((n, r) => n + r.msCount, 0);
|
||||
return {
|
||||
days,
|
||||
source,
|
||||
points: bucketHistory(rows, from, to, days <= 2 ? 96 : days <= KEEP_HOURLY_DAYS ? 200 : 370),
|
||||
availability: checks === 0 ? null : Math.round((okCount / checks) * 1000) / 10,
|
||||
avgMs: msCount === 0 ? null : Math.round(rows.reduce((n, r) => n + r.msSum, 0) / msCount),
|
||||
checks,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── What the history says ─────────────────────────────────────────────────
|
||||
|
||||
@@ -19,7 +19,7 @@ import { getCachedBackupCoverage } from "./backupCoverage.js";
|
||||
import { lastAttempt as lastSelfBackup, listSnapshots } from "./selfBackup.js";
|
||||
import { loadAcks, attentionCounts } from "./alertAcks.js";
|
||||
import { listIncidents } from "./incidents.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
@@ -132,7 +132,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
|
||||
try {
|
||||
for (const s of serverRows) {
|
||||
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), 3650, 101)) {
|
||||
for (const f of disksFillingSoon(await getForecastSeries(s.id, now), 3650, 101)) {
|
||||
out.add("homelab_server_disk_days_to_full", "gauge", "Days until a disk is full at its present rate of filling; only for disks with a steady climb.", f.daysToFull, { server: s.name, mount: f.mount });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* The network equipment read from UniFi controllers: devices with their state, address, uptime and what they are connected through, the networks and the Wi-Fi
|
||||
* names, and which servers are plugged into which device. Shared by the Network devices page and the read API.
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, networkDevices, networkSegments, networkServerLinks, servers } from "../db/schema.js";
|
||||
import { dependentsOf, loadGraph, needsOfNode, nodeKey } from "./dependencies.js";
|
||||
import { activeSubjects } from "./maintenance.js";
|
||||
import { syncStatuses } from "./unifiSync.js";
|
||||
|
||||
export async function buildNetworkOverview() {
|
||||
const [intRows, devices, segments, statuses, graph, silenced, linkRows, serverRows] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "unifi")),
|
||||
db.select().from(networkDevices),
|
||||
db.select().from(networkSegments),
|
||||
syncStatuses(),
|
||||
loadGraph(),
|
||||
activeSubjects(),
|
||||
db.select().from(networkServerLinks),
|
||||
db.select({ id: servers.id, name: servers.name }).from(servers),
|
||||
]);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
const intName = new Map(intRows.map((i) => [i.id, i.name]));
|
||||
const byExternal = new Map(devices.map((d) => [`${d.integrationId}:${d.externalId}`, d]));
|
||||
// The servers plugged into each device, by what the controller says is connected (see unifiSync.matchClientsToServers).
|
||||
const plugged = new Map<number, { serverId: number; serverName: string; port: number | null; wired: boolean }[]>();
|
||||
const serverLinks: { serverId: number; serverName: string; deviceId: number; deviceName: string; port: number | null; wired: boolean }[] = [];
|
||||
for (const l of linkRows) {
|
||||
const device = byExternal.get(`${l.integrationId}:${l.deviceExternalId}`);
|
||||
const name = serverName.get(l.serverId);
|
||||
if (!device || !name) continue;
|
||||
plugged.set(device.id, [...(plugged.get(device.id) ?? []), { serverId: l.serverId, serverName: name, port: l.port, wired: l.wired }]);
|
||||
serverLinks.push({ serverId: l.serverId, serverName: name, deviceId: device.id, deviceName: device.name, port: l.port, wired: l.wired });
|
||||
}
|
||||
return {
|
||||
serverLinks,
|
||||
integrations: intRows.map((i) => ({ id: i.id, name: i.name, enabled: i.enabled, status: statuses[i.id] ?? null })),
|
||||
devices: devices
|
||||
.map((d) => {
|
||||
const key = nodeKey("device", d.id);
|
||||
const uplink = d.uplinkExternalId ? byExternal.get(`${d.integrationId}:${d.uplinkExternalId}`) : undefined;
|
||||
return {
|
||||
id: d.id,
|
||||
integrationId: d.integrationId,
|
||||
integrationName: intName.get(d.integrationId) ?? "UniFi",
|
||||
siteName: d.siteName,
|
||||
name: d.name,
|
||||
model: d.model,
|
||||
kind: d.kind,
|
||||
ip: d.ip,
|
||||
mac: d.mac,
|
||||
state: d.state,
|
||||
uptimeSeconds: d.uptimeSeconds,
|
||||
firmware: d.firmware,
|
||||
firmwareUpdatable: d.firmwareUpdatable,
|
||||
uplink: uplink ? { id: uplink.id, name: uplink.name } : null,
|
||||
offlineSince: d.offlineSince,
|
||||
lastOnlineAt: d.lastOnlineAt,
|
||||
updatedAt: d.updatedAt,
|
||||
// How many things depend on it (directly or through others), and how many it needs: what the dependency map says about it.
|
||||
dependents: dependentsOf(graph.edges, key).length,
|
||||
needs: needsOfNode(graph.edges, key).length,
|
||||
inMaintenance: silenced.has(key),
|
||||
servers: (plugged.get(d.id) ?? []).sort((a, b) => a.serverName.localeCompare(b.serverName)),
|
||||
};
|
||||
})
|
||||
.sort((a, b) => a.name.localeCompare(b.name)),
|
||||
networks: segments.filter((s) => s.kind === "network").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => (a.vlanId ?? 0) - (b.vlanId ?? 0) || a.name.localeCompare(b.name)),
|
||||
ssids: segments.filter((s) => s.kind === "ssid").map(({ kind: _k, ...s }) => ({ ...s, integrationName: intName.get(s.integrationId) ?? "UniFi" })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* What the read API (/api/v1) says. Each of these is a deliberate, stable shape — not the app's own page payloads, which change with the pages — and only things
|
||||
* any signed-in person can already see: no tokens, no secrets, no passwords, no settings. Dates are ISO 8601 in UTC; sizes are bytes; percentages are 0–100.
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, servers } from "../db/schema.js";
|
||||
import { env } from "../env.js";
|
||||
import { attentionCounts, loadAcks } from "./alertAcks.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { loadStats } from "./endpointChecks.js";
|
||||
import { describeTarget, listActiveWindows, listUpcomingWindows } from "./maintenance.js";
|
||||
import { parseHostHealth } from "./hostHealth.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { parseTags } from "./serverTags.js";
|
||||
import { parseThresholds } from "./serverThresholds.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { serverStatus } from "./topology.js";
|
||||
|
||||
/** An address inside the app as a full address, for a script that has no idea where the app lives. */
|
||||
export function absoluteUrl(path: string | null): string | null {
|
||||
return path === null ? null : `${env.appBaseUrl.replace(/\/+$/, "")}${path.startsWith("/") ? "" : "/"}${path}`;
|
||||
}
|
||||
|
||||
const json = <T>(text: string | null, fallback: T): T => {
|
||||
if (!text) return fallback;
|
||||
try {
|
||||
return JSON.parse(text) as T;
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
};
|
||||
|
||||
type ServerRow = typeof servers.$inferSelect;
|
||||
|
||||
export function serverV1(row: ServerRow, offlineMinutes: number, now: Date) {
|
||||
const status = serverStatus(row, offlineMinutes, now);
|
||||
const disks = json<{ mount: string; sizeBytes: number; usedBytes: number }[]>(row.disks, []);
|
||||
const updates = parsePendingUpdates(row.pendingUpdates);
|
||||
const health = parseHostHealth(row.hostHealth);
|
||||
return {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
hostname: row.hostname,
|
||||
description: row.description,
|
||||
os: row.osType,
|
||||
tags: parseTags(row.tags),
|
||||
ipAddresses: json<string[]>(row.ipAddresses, []),
|
||||
// "online" while its agent has reported within its limit, "offline" after, "unknown" if it never has.
|
||||
status: status === "ok" ? "online" : status === "bad" ? "offline" : "unknown",
|
||||
lastSeenAt: row.lastSeenAt,
|
||||
createdAt: row.createdAt,
|
||||
agentVersion: row.agentVersion ?? null,
|
||||
runsOnServerId: row.parentServerId ?? null,
|
||||
hardware: { manufacturer: row.hwManufacturer ?? null, model: row.hwModel ?? null, serial: row.hwSerial ?? null },
|
||||
cpu: { model: row.cpuModel, cores: row.cpuCores, loadPercent: row.cpuLoadPercent },
|
||||
memory: { totalBytes: row.memTotalBytes, usedBytes: row.memUsedBytes },
|
||||
disks: disks.map((d) => ({ mount: d.mount, sizeBytes: d.sizeBytes, usedBytes: d.usedBytes, usedPercent: d.sizeBytes > 0 ? Math.round((d.usedBytes / d.sizeBytes) * 1000) / 10 : null })),
|
||||
updates: updates ? { manager: updates.manager, count: updates.count, security: updates.security ?? null, rebootRequired: !!updates.rebootRequired, reportedAt: updates.reportedAt ?? null } : null,
|
||||
health: health ? { failedUnits: health.failedUnits?.length ?? 0, reportedAt: health.reportedAt ?? null } : null,
|
||||
thresholds: parseThresholds(row.thresholds),
|
||||
url: absoluteUrl(`/servers/${row.id}`),
|
||||
};
|
||||
}
|
||||
|
||||
export async function serversV1(now: Date = new Date()) {
|
||||
const [rows, settings] = await Promise.all([db.select().from(servers).orderBy(servers.name), getSettings()]);
|
||||
return rows.map((r) => serverV1(r, settings.healthChecks.serverOfflineMinutes, now));
|
||||
}
|
||||
|
||||
export async function oneServerV1(id: number, now: Date = new Date()) {
|
||||
const [row] = await db.select().from(servers).where(eq(servers.id, id)).limit(1);
|
||||
if (!row) return null;
|
||||
return serverV1(row, (await getSettings()).healthChecks.serverOfflineMinutes, now);
|
||||
}
|
||||
|
||||
export async function alertsV1() {
|
||||
const [report, acks] = await Promise.all([getAlerts(false), loadAcks()]);
|
||||
const attention = attentionCounts(report.alerts, acks);
|
||||
return {
|
||||
generatedAt: report.generatedAt,
|
||||
// What is on the page: those nobody has looked at, by severity, and how many somebody has.
|
||||
needingAttention: { critical: attention.critical, warning: attention.warning, acknowledged: attention.acknowledged },
|
||||
counts: report.counts,
|
||||
couldntCheck: report.couldntCheck,
|
||||
alerts: report.alerts.map((a) => {
|
||||
const ack = acks.get(a.id);
|
||||
return {
|
||||
id: a.id,
|
||||
severity: a.severity,
|
||||
category: a.category,
|
||||
source: a.source,
|
||||
message: a.message,
|
||||
url: absoluteUrl(a.link),
|
||||
silenced: a.silenced,
|
||||
flapping: !!a.flapping,
|
||||
target: a.target,
|
||||
causedBy: a.causedBy ?? null,
|
||||
affects: a.affects ?? [],
|
||||
acknowledgement: ack?.active ? { by: ack.ackedBy, at: ack.ackedAt, note: ack.note, until: ack.until, assignee: ack.assignee?.name ?? null } : null,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function inventoryV1() {
|
||||
const [rows, serverRows] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select({ id: servers.id, name: servers.name }).from(servers)]);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
return rows.map((a) => ({
|
||||
id: a.id,
|
||||
name: a.name,
|
||||
kind: a.kind,
|
||||
manufacturer: a.manufacturer,
|
||||
model: a.model,
|
||||
serial: a.serial,
|
||||
location: a.location,
|
||||
rackUnit: a.rackUnit,
|
||||
purchaseDate: a.purchaseDate,
|
||||
purchasePrice: a.purchasePrice,
|
||||
warrantyUntil: a.warrantyUntil,
|
||||
endOfSupport: a.endOfSupport,
|
||||
replacementCost: a.replacementCost,
|
||||
powerWatts: a.powerWatts,
|
||||
notes: a.notes,
|
||||
server: a.serverId !== null && serverName.has(a.serverId) ? { id: a.serverId, name: serverName.get(a.serverId)! } : null,
|
||||
updatedAt: a.updatedAt,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function maintenanceV1(now: Date = new Date()) {
|
||||
const shape = async (w: Awaited<ReturnType<typeof listActiveWindows>>[number]) => {
|
||||
const target = await describeTarget(w.targetType, w.targetId);
|
||||
return target ? { id: w.id, target: { type: w.targetType, id: w.targetId, name: target.name, kind: target.kind }, reason: w.reason, startsAt: w.startedAt, endsAt: w.endsAt, createdBy: w.createdBy } : null;
|
||||
};
|
||||
const active = (await Promise.all((await listActiveWindows(now)).map(shape))).filter((w): w is NonNullable<typeof w> => w !== null);
|
||||
const upcoming = (await Promise.all((await listUpcomingWindows(now)).map(shape))).filter((w): w is NonNullable<typeof w> => w !== null);
|
||||
return { active, upcoming };
|
||||
}
|
||||
|
||||
export async function endpointsV1(now: Date = new Date()) {
|
||||
const rows = await db.select().from(endpointChecks).orderBy(endpointChecks.name);
|
||||
const stats = await loadStats(rows.map((r) => r.id), now);
|
||||
return rows.map((r) => {
|
||||
const st = stats.get(r.id);
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
kind: r.kind,
|
||||
target: r.target,
|
||||
port: r.port,
|
||||
enabled: r.enabled,
|
||||
status: !r.enabled ? "paused" : r.lastOk === null ? "unknown" : r.lastOk ? "up" : "down",
|
||||
lastCheckedAt: r.lastCheckedAt,
|
||||
lastResponseMs: r.lastResponseMs,
|
||||
lastMessage: r.lastOk === false ? r.lastMessage : null,
|
||||
certificateExpiresAt: r.certExpiresAt,
|
||||
uptime24hPercent: st?.uptime24h ?? null,
|
||||
uptime7dPercent: st?.uptime7d ?? null,
|
||||
averageResponseMs24h: st?.avgMs24h ?? null,
|
||||
serverId: r.serverId,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -1,19 +1,30 @@
|
||||
/**
|
||||
* A short history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend
|
||||
* A history of what each agent-reported server's load, memory and disks looked like, so a page can show a trend
|
||||
* (and say how soon a filling disk will run out) instead of only the latest figure.
|
||||
*
|
||||
* One sample is kept per server every ten minutes at most, however often its agent reports, for 35 days. Everything here is a
|
||||
* percentage: CPU load (as the agent reports it), memory in use, and each disk's used share.
|
||||
*
|
||||
* Beyond those 35 days the same figures are kept coarser: each sample also goes into its hour (kept four months) and its day (kept over two years), as a
|
||||
* sum, a count, a lowest and a highest. A chart asks for a number of days and is answered from whichever is fine enough and still there: the samples up to
|
||||
* 35 days, the hours up to 120, the days beyond. A year of a disk filling is then one point a day.
|
||||
*/
|
||||
import { and, desc, eq, gte, lt } from "drizzle-orm";
|
||||
import { and, desc, eq, gte, lt, sql } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { serverMetrics } from "../db/schema.js";
|
||||
import { serverMetrics, serverMetricsRollup } from "../db/schema.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
import type { IncomingSystemInfo } from "./taskSync.js";
|
||||
|
||||
/** Reports closer together than this add no sample. */
|
||||
export const MIN_GAP_MS = 10 * 60_000;
|
||||
export const RETENTION_DAYS = 35;
|
||||
/** How long the hourly and daily figures are kept. */
|
||||
export const HOURLY_KEEP_DAYS = 120;
|
||||
export const DAILY_KEEP_DAYS = 800;
|
||||
/** The longest stretch a chart may ask for. */
|
||||
export const MAX_RANGE_DAYS = 365;
|
||||
const DAY_MS = 24 * 3600_000;
|
||||
const HOUR_MS = 3600_000;
|
||||
|
||||
export interface Sample {
|
||||
metric: string;
|
||||
@@ -39,6 +50,35 @@ export function samplesFromSystem(system: IncomingSystemInfo | null | undefined)
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The start of the hour or day a moment falls in, as ISO. */
|
||||
export const hourStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / HOUR_MS) * HOUR_MS).toISOString();
|
||||
export const dayStart = (t: number | string) => new Date(Math.floor(new Date(t).getTime() / DAY_MS) * DAY_MS).toISOString();
|
||||
|
||||
/** Adds samples to their hour and day. Never allowed to break recording the sample itself. */
|
||||
async function addToRollups(serverId: number, samples: Sample[], at: Date): Promise<void> {
|
||||
try {
|
||||
for (const grain of ["h", "d"] as const) {
|
||||
const bucket = grain === "h" ? hourStart(at.getTime()) : dayStart(at.getTime());
|
||||
for (const s of samples) {
|
||||
await db
|
||||
.insert(serverMetricsRollup)
|
||||
.values({ serverId, metric: s.metric, grain, at: bucket, sum: s.value, n: 1, min: s.value, max: s.value })
|
||||
.onConflictDoUpdate({
|
||||
target: [serverMetricsRollup.serverId, serverMetricsRollup.metric, serverMetricsRollup.grain, serverMetricsRollup.at],
|
||||
set: {
|
||||
sum: sql`${serverMetricsRollup.sum} + ${s.value}`,
|
||||
n: sql`${serverMetricsRollup.n} + 1`,
|
||||
min: sql`min(${serverMetricsRollup.min}, ${s.value})`,
|
||||
max: sql`max(${serverMetricsRollup.max}, ${s.value})`,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[metrics] couldn't add to the hourly and daily figures:", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Records a report's figures, unless this server was sampled a moment ago. Returns whether anything was stored. */
|
||||
export async function recordServerMetrics(serverId: number, system: IncomingSystemInfo | null | undefined, now: Date = new Date()): Promise<boolean> {
|
||||
const samples = samplesFromSystem(system);
|
||||
@@ -54,7 +94,32 @@ export async function recordServerMetrics(serverId: number, system: IncomingSyst
|
||||
|
||||
const at = now.toISOString();
|
||||
await db.insert(serverMetrics).values(samples.map((s) => ({ serverId, metric: s.metric, value: s.value, recordedAt: at })));
|
||||
await addToRollups(serverId, samples, now);
|
||||
await db.delete(serverMetrics).where(and(eq(serverMetrics.serverId, serverId), lt(serverMetrics.recordedAt, new Date(now.getTime() - RETENTION_DAYS * DAY_MS).toISOString())));
|
||||
await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "h"), lt(serverMetricsRollup.at, new Date(now.getTime() - HOURLY_KEEP_DAYS * DAY_MS).toISOString())));
|
||||
await db.delete(serverMetricsRollup).where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, "d"), lt(serverMetricsRollup.at, new Date(now.getTime() - DAILY_KEEP_DAYS * DAY_MS).toISOString())));
|
||||
return true;
|
||||
}
|
||||
|
||||
const BACKFILL_FLAG = "metricsRollupBackfilled";
|
||||
|
||||
/**
|
||||
* The first time after the hourly and daily figures were added, builds them from the samples already kept (up to 35 days), so the longer ranges are not empty
|
||||
* for weeks. A bucket is written from what the samples say, replacing what is there, so it is right however far it got before. Done once.
|
||||
*/
|
||||
export async function backfillMetricRollups(): Promise<boolean> {
|
||||
if (await getInternalFlag(BACKFILL_FLAG)) return false;
|
||||
for (const [grain, len] of [["h", 13], ["d", 10]] as const) {
|
||||
const suffix = grain === "h" ? ":00:00.000Z" : "T00:00:00.000Z";
|
||||
await db.run(sql`
|
||||
INSERT INTO server_metrics_rollup (server_id, metric, grain, at, sum, n, min, max)
|
||||
SELECT server_id, metric, ${grain}, substr(recorded_at, 1, ${len}) || ${suffix}, sum(value), count(*), min(value), max(value)
|
||||
FROM server_metrics
|
||||
GROUP BY server_id, metric, substr(recorded_at, 1, ${len})
|
||||
ON CONFLICT (server_id, metric, grain, at) DO UPDATE SET sum = excluded.sum, n = excluded.n, min = excluded.min, max = excluded.max
|
||||
`);
|
||||
}
|
||||
await setInternalFlag(BACKFILL_FLAG, new Date().toISOString());
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -62,6 +127,9 @@ export interface Point {
|
||||
/** ISO time. */
|
||||
t: string;
|
||||
v: number;
|
||||
/** The lowest and highest figure in what this point stands for, when it stands for several (hourly and daily figures). */
|
||||
lo?: number;
|
||||
hi?: number;
|
||||
}
|
||||
|
||||
/** Averages points into at most `maxPoints` equal slices of [from, to], so a month of samples draws as a short line. */
|
||||
@@ -83,6 +151,42 @@ export function bucketPoints(points: { t: number; v: number }[], from: number, t
|
||||
.map(([, b]) => ({ t: new Date(b.tSum / b.n).toISOString(), v: round1(b.sum / b.n) }));
|
||||
}
|
||||
|
||||
/** A row of the hourly or daily figures, with the moment it starts at as epoch ms. */
|
||||
export interface RollupRow {
|
||||
t: number;
|
||||
sum: number;
|
||||
n: number;
|
||||
min: number;
|
||||
max: number;
|
||||
}
|
||||
|
||||
/** The average of what each row stands for. */
|
||||
export function rollupAverages(rows: RollupRow[]): { t: number; v: number }[] {
|
||||
return rows.filter((r) => r.n > 0).map((r) => ({ t: r.t, v: r.sum / r.n }));
|
||||
}
|
||||
|
||||
/** Rows into at most `maxPoints` slices of [from, to]: the average of the samples in each (not of the rows), with the lowest and highest. */
|
||||
export function bucketRollups(rows: RollupRow[], from: number, to: number, maxPoints: number): Point[] {
|
||||
if (rows.length === 0 || to <= from) return [];
|
||||
const size = Math.max((to - from) / maxPoints, 1);
|
||||
const buckets = new Map<number, { sum: number; n: number; lo: number; hi: number; tSum: number; rows: number }>();
|
||||
for (const r of rows) {
|
||||
if (r.t < from || r.t > to || r.n <= 0) continue;
|
||||
const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size));
|
||||
const b = buckets.get(key) ?? { sum: 0, n: 0, lo: Infinity, hi: -Infinity, tSum: 0, rows: 0 };
|
||||
b.sum += r.sum;
|
||||
b.n += r.n;
|
||||
b.lo = Math.min(b.lo, r.min);
|
||||
b.hi = Math.max(b.hi, r.max);
|
||||
b.tSum += r.t;
|
||||
b.rows += 1;
|
||||
buckets.set(key, b);
|
||||
}
|
||||
return [...buckets.entries()]
|
||||
.sort((a, b) => a[0] - b[0])
|
||||
.map(([, b]) => ({ t: new Date(b.tSum / b.rows).toISOString(), v: round1(b.sum / b.n), lo: round1(b.lo), hi: round1(b.hi) }));
|
||||
}
|
||||
|
||||
export interface Trend {
|
||||
/** Change in the percentage per day, from a straight line through the points. */
|
||||
perDay: number;
|
||||
@@ -175,35 +279,86 @@ function describe(metric: string): { kind: MetricSeries["kind"]; label: string }
|
||||
|
||||
const ORDER = { cpu: 0, mem: 1, disk: 2 } as const;
|
||||
|
||||
/** Every series recorded for a server over the last `days`, ready to draw. */
|
||||
/** Which of the kept figures answers a stretch of `days`: the samples while they reach back that far, then the hours, then the days. */
|
||||
export function sourceFor(days: number): "samples" | "hourly" | "daily" {
|
||||
return days <= RETENTION_DAYS ? "samples" : days <= HOURLY_KEEP_DAYS ? "hourly" : "daily";
|
||||
}
|
||||
|
||||
/** How many points a chart over `days` is drawn from at most. */
|
||||
const pointsFor = (days: number) => (days <= RETENTION_DAYS ? 96 : days <= HOURLY_KEEP_DAYS ? 200 : MAX_RANGE_DAYS + 5);
|
||||
|
||||
/** The latest sample of each figure of a server, whatever range is asked for. */
|
||||
async function latestSamples(serverId: number): Promise<Map<string, number>> {
|
||||
const rows = await db.all<{ metric: string; value: number }>(
|
||||
sql`SELECT metric, value FROM server_metrics WHERE id IN (SELECT max(id) FROM server_metrics WHERE server_id = ${serverId} GROUP BY metric)`,
|
||||
);
|
||||
return new Map(rows.map((r) => [r.metric, r.value]));
|
||||
}
|
||||
|
||||
/** Every series recorded for a server over the last `days` (up to a year), ready to draw. */
|
||||
export async function getServerMetrics(serverId: number, days: number, now: Date = new Date()): Promise<MetricSeries[]> {
|
||||
days = Math.min(MAX_RANGE_DAYS, Math.max(1, days));
|
||||
const to = now.getTime();
|
||||
const from = to - days * DAY_MS;
|
||||
const rows = await db
|
||||
.select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt })
|
||||
.from(serverMetrics)
|
||||
.where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString())))
|
||||
.orderBy(serverMetrics.recordedAt);
|
||||
const source = sourceFor(days);
|
||||
|
||||
const byMetric = new Map<string, { t: number; v: number }[]>();
|
||||
for (const r of rows) {
|
||||
const list = byMetric.get(r.metric) ?? [];
|
||||
list.push({ t: Date.parse(r.at), v: r.value });
|
||||
byMetric.set(r.metric, list);
|
||||
// The figures, per metric, as { t, v } for a trend and as points for the chart.
|
||||
const raw = new Map<string, { t: number; v: number }[]>();
|
||||
const points = new Map<string, Point[]>();
|
||||
if (source === "samples") {
|
||||
const rows = await db
|
||||
.select({ metric: serverMetrics.metric, value: serverMetrics.value, at: serverMetrics.recordedAt })
|
||||
.from(serverMetrics)
|
||||
.where(and(eq(serverMetrics.serverId, serverId), gte(serverMetrics.recordedAt, new Date(from).toISOString())))
|
||||
.orderBy(serverMetrics.recordedAt);
|
||||
for (const r of rows) raw.set(r.metric, [...(raw.get(r.metric) ?? []), { t: Date.parse(r.at), v: r.value }]);
|
||||
for (const [metric, list] of raw) points.set(metric, bucketPoints(list, from, to, pointsFor(days)));
|
||||
} else {
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(serverMetricsRollup)
|
||||
.where(and(eq(serverMetricsRollup.serverId, serverId), eq(serverMetricsRollup.grain, source === "hourly" ? "h" : "d"), gte(serverMetricsRollup.at, new Date(from).toISOString())))
|
||||
.orderBy(serverMetricsRollup.at);
|
||||
const byMetric = new Map<string, RollupRow[]>();
|
||||
for (const r of rows) byMetric.set(r.metric, [...(byMetric.get(r.metric) ?? []), { t: Date.parse(r.at), sum: r.sum, n: r.n, min: r.min, max: r.max }]);
|
||||
for (const [metric, list] of byMetric) {
|
||||
raw.set(metric, rollupAverages(list));
|
||||
points.set(metric, bucketRollups(list, from, to, pointsFor(days)));
|
||||
}
|
||||
}
|
||||
const latest = source === "samples" ? null : await latestSamples(serverId);
|
||||
|
||||
return [...byMetric.entries()]
|
||||
.map(([metric, raw]) => {
|
||||
return [...raw.entries()]
|
||||
.map(([metric, list]) => {
|
||||
const { kind, label } = describe(metric);
|
||||
const newest = latest?.get(metric) ?? list[list.length - 1]?.v;
|
||||
return {
|
||||
metric,
|
||||
kind,
|
||||
label,
|
||||
points: bucketPoints(raw, from, to, 96),
|
||||
latest: raw[raw.length - 1]?.v ?? null,
|
||||
points: points.get(metric) ?? [],
|
||||
latest: newest === undefined ? null : source === "samples" ? newest : round1(newest),
|
||||
// A disk filling is the thing worth forecasting; load and memory go up and down by the hour.
|
||||
trend: kind === "disk" ? trendOf(raw) : null,
|
||||
trend: kind === "disk" ? trendOf(list) : null,
|
||||
};
|
||||
})
|
||||
.sort((a, b) => ORDER[a.kind] - ORDER[b.kind] || a.label.localeCompare(b.label));
|
||||
}
|
||||
|
||||
/** How far back the longer look at a disk goes when the last two weeks say it is flat. */
|
||||
export const LONG_FORECAST_DAYS = 90;
|
||||
|
||||
/**
|
||||
* The series to forecast from: the last two weeks, as before — except that a disk those two weeks call flat is looked at again over the last
|
||||
* ninety days, so a slow, steady climb (a point a week) is seen too. The longer trend replaces the short one only if it is really rising.
|
||||
*/
|
||||
export async function getForecastSeries(serverId: number, now: Date = new Date()): Promise<MetricSeries[]> {
|
||||
const short = await getServerMetrics(serverId, 14, now);
|
||||
if (!short.some((s) => s.kind === "disk" && (!s.trend || s.trend.daysToFull === null))) return short;
|
||||
const long = await getServerMetrics(serverId, LONG_FORECAST_DAYS, now);
|
||||
return short.map((s) => {
|
||||
if (s.kind !== "disk" || (s.trend && s.trend.daysToFull !== null)) return s;
|
||||
const longer = long.find((l) => l.metric === s.metric);
|
||||
return longer?.trend && longer.trend.daysToFull !== null ? { ...s, trend: longer.trend } : s;
|
||||
});
|
||||
}
|
||||
@@ -16,7 +16,7 @@ import { describeTarget, listUpcomingWindows } from "./maintenance.js";
|
||||
import { notify, notifyNow } from "./notify.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
@@ -163,7 +163,7 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
const own = parseThresholds(s.thresholds);
|
||||
const forecastDays = effectiveThresholds(settings.healthChecks, own).diskForecastDays;
|
||||
if (!(forecastDays > 0)) continue;
|
||||
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
|
||||
for (const f of disksFillingSoon(await getForecastSeries(s.id, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
|
||||
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user