From f7f5e1cd41ef00db81654852eafe54fb3e10b99a Mon Sep 17 00:00:00 2001 From: Bobban Rydh Date: Thu, 15 Oct 2026 00:12:33 +0200 Subject: [PATCH] Disk fill forecasts, skeleton loaders, and a Silence button on alerts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A disk that is on course to be full within 14 days (setting: 0 turns it off) is flagged on the Alerts page and dashboard, from the server's own history: needs three days of steady growth, skips erratic disks and ones the ordinary disk alert covers. Skeleton placeholders replace "Fetching status…" and "Loading…". Alerts that a maintenance window can silence get a Silence button (1, 4 or 24 hours) that starts one, and the alert list is refreshed at once when a window starts or ends. Co-Authored-By: Claude Sonnet 5.5 --- DATABASE.md | 2 +- README.md | 12 ++ server/src/routes/maintenance.ts | 5 + server/src/routes/settings.ts | 2 +- server/src/services/alertTypes.ts | 2 + server/src/services/alerts.ts | 62 +++++++- server/src/services/serverMetrics.ts | 39 ++++- server/src/services/settingsStore.ts | 4 +- server/test/alertSilence.test.ts | 89 ++++++++++++ server/test/diskForecast.test.ts | 136 ++++++++++++++++++ web/src/api/client.ts | 6 +- web/src/components/IntegrationEditForm.tsx | 5 +- web/src/components/ServerTrends.tsx | 3 +- web/src/components/SilenceButton.tsx | 69 +++++++++ web/src/components/Skeleton.tsx | 52 +++++++ web/src/hooks/useAlertsReport.ts | 18 ++- web/src/pages/Alerts.tsx | 12 +- web/src/pages/Dashboard.tsx | 11 +- web/src/pages/ServerDetail.tsx | 3 +- web/src/pages/settings/BadgeSettings.tsx | 3 +- web/src/pages/settings/DisplaySettings.tsx | 3 +- web/src/pages/settings/LogSettings.tsx | 3 +- .../pages/settings/NotificationSettings.tsx | 23 ++- web/src/styles.css | 27 ++++ 24 files changed, 566 insertions(+), 25 deletions(-) create mode 100644 server/test/alertSilence.test.ts create mode 100644 server/test/diskForecast.test.ts create mode 100644 web/src/components/SilenceButton.tsx create mode 100644 web/src/components/Skeleton.tsx diff --git a/DATABASE.md b/DATABASE.md index 3af6a61..a12ec77 100644 --- a/DATABASE.md +++ b/DATABASE.md @@ -495,7 +495,7 @@ that were never changed aren't stored; the app fills in defaults when reading. | `gotify`, `ntfy`, `smtp`, `webhook` | The four notification channels: enabled, address, and credentials (token / password / secret). The credential fields are stored encrypted (prefix `enc:v1:`) — see [Retention and backups](#retention-and-backups). | | `notifications` | Which events notify (`dnsAdd`, `healthAlerts`, …), the daily-reminder time (`secretCheckTime`, default `08:00`) and `timezone` (default `UTC`), and the integration-failure threshold (default 3). | | `quietHours` | `enabled`, `start`, `end`. | -| `healthChecks` | `serverOfflineMinutes` (60), `diskUsagePercent` (90), `domainWarnDays` (30). | +| `healthChecks` | `serverOfflineMinutes` (60), `diskUsagePercent` (90), `domainWarnDays` (30), `diskForecastDays` (14; 0 = off — how far ahead a filling disk is flagged on the Alerts page). | | `logRetention` | `enabled`, `retentionDays` (90), `intervalHours` (24). | | `display` | `dateFormat`, `timeFormat`, `pageSize`. | | `providerColors`, `integrationColors` | Badge colours, by provider / integration type. | diff --git a/README.md b/README.md index 59c5ffb..7274e80 100644 --- a/README.md +++ b/README.md @@ -310,6 +310,18 @@ tickets, and integrations whose calls keep failing. It runs the same checks that send the notifications — so the two can't disagree — but ignores the on/off toggles, since it's for looking at rather than being interrupted by. Problems under a maintenance window stay listed, marked silenced and counted separately. +It also looks ahead: from an agent server's own history (see its Trends card), a +disk that is on course to be full within 14 days (Settings → Notifications → "Warn +when a disk will fill within"; 0 turns it off) is flagged with its rate of growth +and the days left — critical when that is three days or less. It needs at least +three days of steady growth to say anything, skips a disk that jumped once or +wobbles, and leaves out one already past the usual disk threshold, which has its +own alert. These forecasts appear on the page and the dashboard (and in alert +tickets if you've chosen disk problems); they are not sent as notifications. On +any alert that a maintenance window can silence, operators and admins get a +**Silence** button (1, 4 or 24 hours): it starts a maintenance window on that +server or integration — so everything about it goes quiet, which the toast says — +and it can be ended early under Maintenance. It checks live (a recent result is reused for a minute; "Check now" forces a fresh one), and anything it couldn't read is called out at the top rather than quietly treated as fine. diff --git a/server/src/routes/maintenance.ts b/server/src/routes/maintenance.ts index 139e0ac..50fe91d 100644 --- a/server/src/routes/maintenance.ts +++ b/server/src/routes/maintenance.ts @@ -4,6 +4,7 @@ import { z } from "zod"; import { db } from "../db/client.js"; import { dnsProviders, integrations, maintenanceAliases, maintenanceTargetTypes, maintenanceWindows, servers } from "../db/schema.js"; import { hostOfAddress } from "../services/kumaTargets.js"; +import { invalidateAlertsCache } from "../services/alerts.js"; import { requireAuth, requireRole } from "../auth/middleware.js"; import { recordAudit } from "../services/audit.js"; import { describeTarget, listActiveWindows, listUpcomingWindows, startOrExtendWindow } from "../services/maintenance.js"; @@ -80,6 +81,7 @@ maintenanceRouter.post("/", requireRole("operator"), asyncHandler(async (req, re detail: { name: result.target.name, minutes, reason: reason ?? null }, }); + invalidateAlertsCache(); res.status(201).json({ window: { ...result.window, targetName: result.target.name, targetKind: result.target.kind } }); })); @@ -149,6 +151,7 @@ maintenanceRouter.post("/import/uptimekuma", requireRole("operator"), asyncHandl imported.push({ serverId: m.matchedServer.id, serverName: result.target.name, monitorName: m.name, extended: result.extended }); } + invalidateAlertsCache(); res.json({ imported, unmatched, monitorsInMaintenance: inMaintenance.length, monitorsChecked: monitors.length }); })); @@ -223,6 +226,7 @@ maintenanceRouter.post("/sync/uptimekuma", requireRole("operator"), asyncHandler targetId: parsed.data.integrationId, detail: { started: result.started, updated: result.updated, removed: result.removed }, }); + invalidateAlertsCache(); res.json(result); } catch (err) { if (err instanceof KumaSyncError) { @@ -253,5 +257,6 @@ maintenanceRouter.delete("/:id", requireRole("operator"), asyncHandler(async (re targetId: existing.targetId, detail: { name: target?.name ?? null }, }); + invalidateAlertsCache(); res.status(204).end(); })); diff --git a/server/src/routes/settings.ts b/server/src/routes/settings.ts index 9d89f5b..13e3d49 100644 --- a/server/src/routes/settings.ts +++ b/server/src/routes/settings.ts @@ -92,7 +92,7 @@ const updateSchema = z.object({ .partial() .optional(), healthChecks: z - .object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365) }) + .object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90) }) .partial() .optional(), quietHours: z diff --git a/server/src/services/alertTypes.ts b/server/src/services/alertTypes.ts index 204993b..00558bd 100644 --- a/server/src/services/alertTypes.ts +++ b/server/src/services/alertTypes.ts @@ -18,6 +18,8 @@ export interface Alert { link: string | null; /** Under an active maintenance window: still a real problem, but notifications for it are held back. */ silenced: boolean; + /** What a maintenance window would have to be on to silence it, when there is such a thing — which is what "Silence" acts on. */ + target: { type: "server" | "integration" | "dns_provider"; id: number; name: string } | null; } /** One integration that couldn't be read while collecting — so the page never mistakes "couldn't check" for "all clear". */ diff --git a/server/src/services/alerts.ts b/server/src/services/alerts.ts index 2c9b394..74e5cb3 100644 --- a/server/src/services/alerts.ts +++ b/server/src/services/alerts.ts @@ -11,7 +11,8 @@ */ import { eq } from "drizzle-orm"; import { db } from "../db/client.js"; -import { integrations, secrets } from "../db/schema.js"; +import { integrations, secrets, servers } from "../db/schema.js"; +import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js"; import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js"; import { createOsTicketAdapter } from "../integrations/osticket/adapter.js"; @@ -95,8 +96,26 @@ export async function collectAlerts(): Promise { const subjects = await activeSubjects(); const intRows = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations); const intById = new Map(intRows.map((r) => [r.id, r])); + const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((r) => [r.id, r.name])); - function push(a: { category: AlertCategory; severity: AlertSeverity; type: string; message: string; key: string; link?: string | null; silenced?: boolean }) { + /** The thing a maintenance window would go on to silence an alert about `subject` ("server:3", "integration:1"). */ + const targetOf = (type: "server" | "integration" | "dns_provider", id: number): Alert["target"] => ({ + type, + id, + name: (type === "server" ? serverNames.get(id) : type === "integration" ? intById.get(id)?.name : undefined) ?? `${type} #${id}`, + }); + const targetOfSubject = (subject: string | null): Alert["target"] => { + const m = /^(server|integration|dns_provider):(\d+)$/.exec(subject ?? ""); + return m ? targetOf(m[1] as "server" | "integration" | "dns_provider", Number(m[2])) : null; + }; + const integrationTarget = (id: number): Alert["target"] => targetOf("integration", id); + /** Integration-down alerts are per kind of service; a window on any integration of that kind silences them, so offer the first. */ + const firstIntegrationOfType = (type: string): Alert["target"] => { + const row = intRows.find((r) => r.type === type && r.enabled); + return row ? integrationTarget(row.id) : null; + }; + + function push(a: { category: AlertCategory; severity: AlertSeverity; type: string; message: string; key: string; link?: string | null; silenced?: boolean; target?: Alert["target"] }) { const meta = SOURCES[a.type]; alerts.push({ id: `${a.category}:${a.key}`, @@ -106,6 +125,7 @@ export async function collectAlerts(): Promise { message: a.message, link: a.link === undefined ? (meta?.link ?? null) : a.link, silenced: !!a.silenced, + target: a.target ?? null, }); } @@ -150,7 +170,7 @@ export async function collectAlerts(): Promise { message: c.message, key: c.key, link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}` : undefined, - silenced: subject !== null && subjects.has(subject), + silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject), }); } // Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.) @@ -167,7 +187,7 @@ export async function collectAlerts(): Promise { for (const c of evaluateAutomation(items).conditions) { const origin = originOfConditionKey(c.key); const subject = subjectOfConditionKey(c.key); - push({ category: "automation", severity: "warning", type: origin.type, message: c.message, key: c.key, silenced: subject !== null && subjects.has(subject) }); + push({ category: "automation", severity: "warning", type: origin.type, message: c.message, key: c.key, silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject) }); } for (const h of held) { const m = /^(semaphore|gitea):(\d+)$/.exec(h); @@ -187,6 +207,7 @@ export async function collectAlerts(): Promise { message: `Latest backup on ${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} didn't succeed [${f.integrationName}]: ${f.status}`, key: `proxmox-backup:${f.integrationId}:${f.node}`, silenced: f.silenced, + target: integrationTarget(f.integrationId), }); } for (const u of uncovered) { @@ -197,6 +218,7 @@ export async function collectAlerts(): Promise { message: `${u.guestName} (#${u.vmid}) on ${u.node} isn't covered by any backup job [${u.integrationName}]`, key: `proxmox-uncovered:${u.integrationId}:${u.vmid}`, silenced: u.silenced, + target: integrationTarget(u.integrationId), }); } sourceFailures.forEach((f) => cannotCheckIntegration(f, "Proxmox backups")); @@ -214,6 +236,7 @@ export async function collectAlerts(): Promise { : `Datastore "${p.datastore}" has ${plural(p.failedCount, "snapshot")} that failed verification [${p.integrationName}]`, key: `pbs:${p.integrationId}:${p.datastore}`, silenced: p.silenced, + target: integrationTarget(p.integrationId), }); } sourceFailures.forEach((f) => cannotCheckIntegration(f, "Backup verification")); @@ -229,6 +252,7 @@ export async function collectAlerts(): Promise { message: `${u.containerName} [${u.environmentName}, ${u.integrationName}] has an image update available${u.newerVersion ? ` → ${u.newerVersion}` : ""}`, key: `docker:${u.integrationId}:${u.environmentName}:${u.containerName}`, silenced: silencedIntegration(u.integrationId), + target: integrationTarget(u.integrationId), }); } failures.forEach((f) => cannotCheckIntegration(f, "Image updates")); @@ -244,6 +268,7 @@ export async function collectAlerts(): Promise { message: k.daysLeft < 0 ? `Key for ${k.deviceLabel} [${k.integrationName}] has expired` : `Key for ${k.deviceLabel} [${k.integrationName}] expires in ${plural(k.daysLeft, "day")}`, key: `tailscale-key:${k.integrationId}:${k.deviceLabel}`, silenced: silencedIntegration(k.integrationId), + target: integrationTarget(k.integrationId), }); } failures.forEach((f) => cannotCheckIntegration(f, "Tailscale keys")); @@ -263,6 +288,7 @@ export async function collectAlerts(): Promise { message: `Monitor "${m.name}" is down${m.target ? ` (${m.target}${m.port ? `:${m.port}` : ""})` : ""} [${row.name}]`, key: `kuma:${row.id}:${m.id}`, silenced: silencedIntegration(row.id), + target: integrationTarget(row.id), }); } } catch (err) { @@ -285,6 +311,7 @@ export async function collectAlerts(): Promise { message: `${plural(overdue, "open ticket")} ${overdue === 1 ? "is" : "are"} overdue [${row.name}]`, key: `osticket:${row.id}`, silenced: silencedIntegration(row.id), + target: integrationTarget(row.id), }); } } catch (err) { @@ -329,6 +356,27 @@ export async function collectAlerts(): Promise { } }), + // Not a notification: a disk that is on course to fill is something to see coming, so it is shown here, from the server's own history. + check("Disk forecasts", async () => { + const { healthChecks } = await getSettings(); + if (!(healthChecks.diskForecastDays > 0)) return; + for (const s of await db.select({ id: servers.id, name: servers.name }).from(servers)) { + const series = await getServerMetrics(s.id, 14); + for (const f of disksFillingSoon(series, healthChecks.diskForecastDays, healthChecks.diskUsagePercent)) { + push({ + category: "disk", + severity: f.daysToFull <= 3 ? "critical" : "warning", + type: "server", + message: `Disk ${f.mount} on ${s.name} is filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")} (${Math.round(f.latest)}% used now)`, + key: `disk-forecast:${s.id}:${f.mount}`, + link: `/servers/${s.id}`, + silenced: subjects.has(`server:${s.id}`), + target: targetOf("server", s.id), + }); + } + } + }), + check("Integration failures", async () => { const { notifications } = await getSettings(); for (const f of getFailingSources()) { @@ -340,6 +388,7 @@ export async function collectAlerts(): Promise { key: `failing:${f.source}`, link: null, silenced: await isSourceInMaintenance(f.source), + target: firstIntegrationOfType(f.source), }); } }), @@ -369,6 +418,11 @@ export async function collectAlerts(): Promise { } let cache: { at: number; report: AlertsReport } | null = null; + +/** Forget the stored result, so the next look is a fresh one — after something changes what it would say (a maintenance window starting or ending). */ +export function invalidateAlertsCache(): void { + cache = null; +} let inFlight: Promise | null = null; /** The current alerts, reusing a recent result unless `force` asks for a fresh one (and even then not more than once every few seconds). */ diff --git a/server/src/services/serverMetrics.ts b/server/src/services/serverMetrics.ts index 4f01ef7..010bd05 100644 --- a/server/src/services/serverMetrics.ts +++ b/server/src/services/serverMetrics.ts @@ -86,6 +86,10 @@ export function bucketPoints(points: { t: number; v: number }[], from: number, t export interface Trend { /** Change in the percentage per day, from a straight line through the points. */ perDay: number; + /** How well a straight line describes the points, 0 to 1 (R²). A steady climb is near 1; a single jump or a wobble is low. */ + fit: number; + /** How much time the points cover, in days. */ + spanDays: number; /** For something that fills up: days until it reaches 100% at that rate. Null when it isn't rising or the rate is negligible. */ daysToFull: number | null; } @@ -106,16 +110,49 @@ export function trendOf(points: { t: number; v: number }[]): Trend | null { const meanY = ys.reduce((a, b) => a + b, 0) / n; let num = 0; let den = 0; + let totalVariance = 0; for (let i = 0; i < n; i++) { num += (xs[i] - meanX) * (ys[i] - meanY); den += (xs[i] - meanX) ** 2; + totalVariance += (ys[i] - meanY) ** 2; } if (den === 0) return null; const slope = num / den; + // R²: the share of the movement the straight line explains. A flat series has nothing to explain, so it is no "fit" at all. + const fit = totalVariance === 0 ? 0 : Math.round(((num * num) / (den * totalVariance)) * 100) / 100; const perDay = Math.round(slope * 100) / 100; const last = ys[n - 1]; const daysToFull = perDay >= 0.05 && last < 100 ? Math.ceil((100 - last) / perDay) : null; - return { perDay, daysToFull }; + return { perDay, fit, spanDays: Math.round((span / DAY_MS) * 10) / 10, daysToFull }; +} + +export interface DiskForecast { + mount: string; + latest: number; + perDay: number; + daysToFull: number; +} + +/** What a forecast needs before it is worth raising an alert about: enough history, and growth that really is steady. */ +export const FORECAST_MIN_SPAN_DAYS = 3; +export const FORECAST_MIN_FIT = 0.9; + +/** + * The disks that are heading for full within `withinDays`. A disk already at or above `alreadyAlertedAt` percent is left out — the + * ordinary "disk nearly full" alert covers it — and so is one whose growth is erratic (a big copy, a log that rotates), where a + * straight-line forecast would just be a guess. + */ +export function disksFillingSoon(series: MetricSeries[], withinDays: number, alreadyAlertedAt: number): DiskForecast[] { + const out: DiskForecast[] = []; + for (const s of series) { + if (s.kind !== "disk" || !s.trend || s.latest === null) continue; + const { perDay, fit, spanDays, daysToFull } = s.trend; + if (daysToFull === null || daysToFull > withinDays) continue; + if (spanDays < FORECAST_MIN_SPAN_DAYS || fit < FORECAST_MIN_FIT) continue; + if (s.latest >= alreadyAlertedAt) continue; + out.push({ mount: s.metric.slice("disk:".length), latest: s.latest, perDay, daysToFull }); + } + return out.sort((a, b) => a.daysToFull - b.daysToFull); } export interface MetricSeries { diff --git a/server/src/services/settingsStore.ts b/server/src/services/settingsStore.ts index 9207bcf..efdca77 100644 --- a/server/src/services/settingsStore.ts +++ b/server/src/services/settingsStore.ts @@ -95,6 +95,8 @@ export interface HealthCheckSettings { diskUsagePercent: number; /** A domain registration expiring within this many days is reminded about daily (same schedule as secret expiry). */ domainWarnDays: number; + /** An agent-reported disk that is on course to be full within this many days is flagged on the Alerts page. 0 turns it off. */ + diskForecastDays: number; } export interface ConsistencySettings { @@ -180,7 +182,7 @@ const DEFAULTS: AppSettings = { display: { dateFormat: "ymd", timeFormat: "24h", pageSize: 20 }, logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 }, quietHours: { enabled: false, start: "22:00", end: "07:00" }, - healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30 }, + healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 }, // Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every // Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN. consistency: { excludedRanges: ["172.16.0.0/12"] }, diff --git a/server/test/alertSilence.test.ts b/server/test/alertSilence.test.ts new file mode 100644 index 0000000..e037bed --- /dev/null +++ b/server/test/alertSilence.test.ts @@ -0,0 +1,89 @@ +/** + * The Silence button on an alert: the alert says what a maintenance window would have to be on, and starting one through the + * maintenance route shows in the very next look at the alerts (not a minute later). + * + * Run with: npm test --workspace server + */ +import { test, describe, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AddressInfo } from "node:net"; +import type { Server } from "node:http"; + +const dir = mkdtempSync(join(tmpdir(), "hm-silence-")); +process.env.DATABASE_PATH = join(dir, "test.sqlite"); +process.env.SESSION_DIR = join(dir, "sessions"); +process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); + +const express = (await import("express")).default; +const { db } = await import("../src/db/client.js"); +const { users, servers, serverMetrics } = await import("../src/db/schema.js"); +const { runMigrations } = await import("../src/db/migrate.js"); +const { getAlerts } = await import("../src/services/alerts.js"); +const { maintenanceRouter } = await import("../src/routes/maintenance.js"); +const { updateSettings } = await import("../src/services/settingsStore.js"); + +await runMigrations(); +const DAY = 24 * 3600_000; + +describe("silencing an alert", () => { + let app: Server; + let base = ""; + let serverId = 0; + + before(async () => { + await db.insert(users).values({ oidcSub: "tester", name: "Tester", role: "operator" }); + serverId = (await db.insert(servers).values({ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a" }).returning())[0].id; + const now = Date.now(); + for (let i = 0; i <= 32; i++) { + await db.insert(serverMetrics).values({ serverId, metric: "disk:/data", value: 50 + i * 0.5, recordedAt: new Date(now - (32 - i) * 0.25 * DAY).toISOString() }); + } + await updateSettings({ healthChecks: { diskForecastDays: 30 } }); + + const a = express(); + a.use(express.json()); + a.use((req, _res, next) => { + (req as unknown as { session: unknown }).session = { user: { sub: "tester" } }; + next(); + }); + a.use("/api/maintenance", maintenanceRouter); + app = await new Promise((resolve) => { + const s = a.listen(0, "127.0.0.1", () => resolve(s)); + }); + base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/maintenance`; + }); + + after(() => { + app.close(); + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // Windows keeps the database file locked until the process exits. + } + }); + + const forecast = async () => (await getAlerts(false)).alerts.find((a) => a.id.startsWith("disk:disk-forecast:"))!; + + test("the alert names what a window would go on", async () => { + const alert = await forecast(); + assert.deepEqual(alert.target, { type: "server", id: serverId, name: "elsa" }); + assert.equal(alert.silenced, false); + }); + + test("starting a window through the route shows in the next look, and ending it does too", async () => { + const alert = await forecast(); + const started = await fetch(base, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ targetType: alert.target!.type, targetId: alert.target!.id, minutes: 60, reason: "Silenced from the Alerts page" }), + }); + assert.equal(started.status, 201); + const window = (await started.json()).window; + assert.equal((await forecast()).silenced, true, "silenced at once — the stored result was dropped"); + + assert.equal((await fetch(`${base}/${window.id}`, { method: "DELETE" })).status, 204); + assert.equal((await forecast()).silenced, false, "and active again at once when it is ended"); + }); +}); diff --git a/server/test/diskForecast.test.ts b/server/test/diskForecast.test.ts new file mode 100644 index 0000000..24c67d1 --- /dev/null +++ b/server/test/diskForecast.test.ts @@ -0,0 +1,136 @@ +/** + * "This disk will be full in N days" on the Alerts page, from a server's own history. + * + * Run with: npm test --workspace server + */ +import { test, describe, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const dir = mkdtempSync(join(tmpdir(), "hm-forecast-")); +process.env.DATABASE_PATH = join(dir, "test.sqlite"); +process.env.SESSION_DIR = join(dir, "sessions"); +process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); + +const { eq } = await import("drizzle-orm"); +const { db } = await import("../src/db/client.js"); +const { servers, serverMetrics, maintenanceWindows } = await import("../src/db/schema.js"); +const { runMigrations } = await import("../src/db/migrate.js"); +const { collectAlerts } = await import("../src/services/alerts.js"); +const { updateSettings } = await import("../src/services/settingsStore.js"); +const { trendOf, disksFillingSoon } = await import("../src/services/serverMetrics.js"); + +await runMigrations(); + +const DAY = 24 * 3600_000; + +describe("trendOf: how well a straight line fits", () => { + const steady = Array.from({ length: 30 }, (_, i) => ({ t: i * 0.25 * DAY, v: 40 + i * 0.5 + (i % 2 ? 0.2 : -0.2) })); + const jump = Array.from({ length: 30 }, (_, i) => ({ t: i * 0.25 * DAY, v: i < 15 ? 40 : 70 })); + + test("a steady climb fits well; a single jump does not", () => { + assert.ok(trendOf(steady)!.fit > 0.95); + assert.ok(trendOf(jump)!.fit < 0.8); + }); + + test("it says how long a time it covers", () => { + assert.equal(trendOf(steady)!.spanDays, 7.3); + }); +}); + +describe("disksFillingSoon", () => { + const series = (over: Partial<{ latest: number; perDay: number; fit: number; spanDays: number; daysToFull: number | null }> = {}) => { + const t = { perDay: 2, fit: 0.95, spanDays: 7, daysToFull: 10, ...over }; + return [{ metric: "disk:/data", kind: "disk" as const, label: "Disk /data", points: [], latest: over.latest ?? 80, trend: t }]; + }; + + test("picks a disk that fills within the window", () => { + assert.deepEqual(disksFillingSoon(series(), 14, 90).map((d) => d.mount), ["/data"]); + }); + + test("leaves out one that is later, already past the usual threshold, short of history, or erratic", () => { + assert.equal(disksFillingSoon(series({ daysToFull: 30 }), 14, 90).length, 0, "later than the window"); + assert.equal(disksFillingSoon(series({ latest: 92 }), 14, 90).length, 0, "the ordinary disk alert covers it"); + assert.equal(disksFillingSoon(series({ spanDays: 1.5 }), 14, 90).length, 0, "too little history"); + assert.equal(disksFillingSoon(series({ fit: 0.3 }), 14, 90).length, 0, "growth is erratic"); + assert.equal(disksFillingSoon(series({ daysToFull: null }), 14, 90).length, 0, "not rising"); + }); +}); + +describe("the alert", () => { + let steadyId = 0; + let jumpyId = 0; + let fullId = 0; + + /** A disk sampled every 6 hours for 8 days ending now. */ + async function history(serverId: number, mount: string, valueAt: (i: number) => number) { + const now = Date.now(); + for (let i = 0; i <= 32; i++) { + await db.insert(serverMetrics).values({ serverId, metric: `disk:${mount}`, value: valueAt(i), recordedAt: new Date(now - (32 - i) * 0.25 * DAY).toISOString() }); + } + } + + before(async () => { + steadyId = (await db.insert(servers).values({ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a" }).returning())[0].id; + jumpyId = (await db.insert(servers).values({ name: "jumpy", apiTokenHash: "b", apiTokenPrefix: "b" }).returning())[0].id; + fullId = (await db.insert(servers).values({ name: "nearlyfull", apiTokenHash: "c", apiTokenPrefix: "c" }).returning())[0].id; + // 2 points a day: 50% a week ago... 66% now → 17 days to full at that rate. + await history(steadyId, "/data", (i) => 50 + i * 0.5); + await history(jumpyId, "/data", (i) => (i < 16 ? 50 : 72)); + // Already past the 90% disk alert, and still climbing. + await history(fullId, "/data", (i) => 88 + i * 0.1); + await updateSettings({ healthChecks: { diskForecastDays: 30 } }); + }); + + after(() => { + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // Windows keeps the database file locked until the process exits. + } + }); + + const forecasts = async () => (await collectAlerts()).alerts.filter((a) => a.id.startsWith("disk:disk-forecast:")); + + test("a steadily filling disk is flagged, with the rate, the days left and where to look", async () => { + const found = await forecasts(); + assert.equal(found.length, 1, found.map((a) => a.message).join(" | ")); + assert.match(found[0].message, /Disk \/data on elsa/); + assert.match(found[0].message, /about 2\.0 points a day/); + assert.match(found[0].message, /full in about 17 days/); + assert.equal(found[0].link, `/servers/${steadyId}`); + assert.equal(found[0].severity, "warning"); + assert.deepEqual(found[0].target, { type: "server", id: steadyId, name: "elsa" }); + assert.equal(found[0].silenced, false); + }); + + test("a disk that jumped once, and one the ordinary alert already covers, are not flagged", async () => { + const names = (await forecasts()).map((a) => a.message).join(" "); + assert.ok(!names.includes("jumpy")); + assert.ok(!names.includes("nearlyfull")); + }); + + test("a window on the server silences it", async () => { + const now = new Date(); + await db.insert(maintenanceWindows).values({ targetType: "server", targetId: steadyId, startedAt: new Date(now.getTime() - 3600_000).toISOString(), endsAt: new Date(now.getTime() + 3600_000).toISOString() }); + const [alert] = await forecasts(); + assert.equal(alert.silenced, true); + await db.delete(maintenanceWindows); + }); + + test("a disk that fills within three days is critical", async () => { + await db.delete(serverMetrics).where(eq(serverMetrics.serverId, steadyId)); + await history(steadyId, "/data", (i) => 60 + i * 1.1); // 4.4 points a day from ~95% — full in a day or so + await updateSettings({ healthChecks: { diskUsagePercent: 99 } }); + const [alert] = await forecasts(); + assert.equal(alert.severity, "critical"); + await updateSettings({ healthChecks: { diskUsagePercent: 90 } }); + }); + + test("with the setting at 0 nothing is flagged", async () => { + await updateSettings({ healthChecks: { diskForecastDays: 0 } }); + assert.equal((await forecasts()).length, 0); + }); +}); diff --git a/web/src/api/client.ts b/web/src/api/client.ts index c667608..f3e72fb 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -133,7 +133,7 @@ export interface ServerMetricSeries { points: { t: string; v: number }[]; latest: number | null; /** Disks only: the rate of change in points a day, and the days until full when it is rising. */ - trend: { perDay: number; daysToFull: number | null } | null; + trend: { perDay: number; fit: number; spanDays: number; daysToFull: number | null } | null; } export interface IpamEntry { @@ -282,6 +282,8 @@ export interface HealthCheckSettings { serverOfflineMinutes: number; diskUsagePercent: number; domainWarnDays: number; + /** A disk on course to be full within this many days is flagged on the Alerts page; 0 is off. */ + diskForecastDays: number; } export interface QuietHoursSettings { @@ -434,6 +436,8 @@ export interface AlertItem { link: string | null; /** Under a maintenance window: still a real problem, but its notifications are held back. */ silenced: boolean; + /** What a maintenance window would go on to silence it — what the Silence button acts on. Null when nothing can silence it. */ + target?: { type: MaintenanceTargetType; id: number; name: string } | null; /** The osTicket ticket opened for it, if one was. */ ticket?: { number: string | null; url: string | null; grouped: boolean; openedAt: string | null } | null; } diff --git a/web/src/components/IntegrationEditForm.tsx b/web/src/components/IntegrationEditForm.tsx index 1ded624..51882bb 100644 --- a/web/src/components/IntegrationEditForm.tsx +++ b/web/src/components/IntegrationEditForm.tsx @@ -1,5 +1,6 @@ import { useEffect, useState } from "react"; import { api, type IntegrationField, type IntegrationSummary, type IntegrationType } from "../api/client"; +import { BlockSkeleton } from "./Skeleton"; const TYPE_LABELS: Record = { tailscale: "Tailscale", @@ -93,7 +94,9 @@ export default function IntegrationEditForm({ if (loading) { return (
-
Loading…
+
+ +
); } diff --git a/web/src/components/ServerTrends.tsx b/web/src/components/ServerTrends.tsx index 7e90c4e..c005bb3 100644 --- a/web/src/components/ServerTrends.tsx +++ b/web/src/components/ServerTrends.tsx @@ -1,6 +1,7 @@ import { useEffect, useState } from "react"; import { api, type ServerMetricSeries } from "../api/client"; import Sparkline from "./Sparkline"; +import { BlockSkeleton } from "./Skeleton"; const RANGES = [ { days: 1, label: "24 hours" }, @@ -64,7 +65,7 @@ export default function ServerTrends({ serverId }: { serverId: number }) { {failed ? (
Couldn't load the history.
) : series === null ? ( -
Loading…
+ ) : series.length === 0 ? (
No history yet. A sample is kept about every ten minutes once the agent has reported, so lines appear here after a short while. diff --git a/web/src/components/SilenceButton.tsx b/web/src/components/SilenceButton.tsx new file mode 100644 index 0000000..4d6d62a --- /dev/null +++ b/web/src/components/SilenceButton.tsx @@ -0,0 +1,69 @@ +import { useState } from "react"; +import { api, type AlertItem } from "../api/client"; +import { readableError } from "../utils/errors"; +import { toast } from "../utils/toast"; + +const CHOICES = [ + { minutes: 60, label: "1 h", words: "1 hour" }, + { minutes: 240, label: "4 h", words: "4 hours" }, + { minutes: 1440, label: "24 h", words: "24 hours" }, +]; + +/** + * "Silence" on an alert: pick how long, and a maintenance window goes on whatever the alert is about. That silences everything + * about that server or integration, not just this one alert — which is what a maintenance window is — so it says so, and the + * Maintenance page is where to end it early. The choices open in place, so nothing is clipped by the table around it. + */ +export default function SilenceButton({ alert, onDone }: { alert: AlertItem; onDone: () => void }) { + const [choosing, setChoosing] = useState(false); + const [busy, setBusy] = useState(false); + const target = alert.target; + if (!target || alert.silenced) return null; + + async function silence(minutes: number, words: string) { + if (!target) return; + setBusy(true); + try { + await api.maintenance.start({ + targetType: target.type, + targetId: target.id, + minutes, + reason: `Silenced from the Alerts page: ${alert.message}`.slice(0, 200), + }); + toast.success(`${target.name} is silenced for ${words} — every alert about it. End it early under Maintenance.`); + // The sidebar badge and the maintenance banner listen for this. + window.dispatchEvent(new Event("maintenance-changed")); + setChoosing(false); + onDone(); + } catch (err) { + toast.error(readableError(err)); + } finally { + setBusy(false); + } + } + + if (!choosing) { + return ( + + ); + } + return ( +
+ {CHOICES.map((c) => ( + + ))} + +
+ ); +} diff --git a/web/src/components/Skeleton.tsx b/web/src/components/Skeleton.tsx new file mode 100644 index 0000000..2fabe9b --- /dev/null +++ b/web/src/components/Skeleton.tsx @@ -0,0 +1,52 @@ +import type { CSSProperties } from "react"; + +/** A grey bar with a slow shimmer, standing where text or a figure will be, so the page keeps its shape while it loads. */ +export function Skeleton({ width = "100%", height = "1rem", className = "", style }: { width?: string | number; height?: string | number; className?: string; style?: CSSProperties }) { + return