Disk fill forecasts, skeleton loaders, and a Silence button on alerts
A disk that is on course to be full within 14 days (setting: 0 turns it off) is flagged on the Alerts page and dashboard, from the server's own history: needs three days of steady growth, skips erratic disks and ones the ordinary disk alert covers. Skeleton placeholders replace "Fetching status…" and "Loading…". Alerts that a maintenance window can silence get a Silence button (1, 4 or 24 hours) that starts one, and the alert list is refreshed at once when a window starts or ends. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
82974b98d0
commit
f7f5e1cd41
24 files changed
+566
-25
No files matched your search
@@ -4,6 +4,7 @@ import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { dnsProviders, integrations, maintenanceAliases, maintenanceTargetTypes, maintenanceWindows, servers } from "../db/schema.js";
|
||||
import { hostOfAddress } from "../services/kumaTargets.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { describeTarget, listActiveWindows, listUpcomingWindows, startOrExtendWindow } from "../services/maintenance.js";
|
||||
@@ -80,6 +81,7 @@ maintenanceRouter.post("/", requireRole("operator"), asyncHandler(async (req, re
|
||||
detail: { name: result.target.name, minutes, reason: reason ?? null },
|
||||
});
|
||||
|
||||
invalidateAlertsCache();
|
||||
res.status(201).json({ window: { ...result.window, targetName: result.target.name, targetKind: result.target.kind } });
|
||||
}));
|
||||
|
||||
@@ -149,6 +151,7 @@ maintenanceRouter.post("/import/uptimekuma", requireRole("operator"), asyncHandl
|
||||
imported.push({ serverId: m.matchedServer.id, serverName: result.target.name, monitorName: m.name, extended: result.extended });
|
||||
}
|
||||
|
||||
invalidateAlertsCache();
|
||||
res.json({ imported, unmatched, monitorsInMaintenance: inMaintenance.length, monitorsChecked: monitors.length });
|
||||
}));
|
||||
|
||||
@@ -223,6 +226,7 @@ maintenanceRouter.post("/sync/uptimekuma", requireRole("operator"), asyncHandler
|
||||
targetId: parsed.data.integrationId,
|
||||
detail: { started: result.started, updated: result.updated, removed: result.removed },
|
||||
});
|
||||
invalidateAlertsCache();
|
||||
res.json(result);
|
||||
} catch (err) {
|
||||
if (err instanceof KumaSyncError) {
|
||||
@@ -253,5 +257,6 @@ maintenanceRouter.delete("/:id", requireRole("operator"), asyncHandler(async (re
|
||||
targetId: existing.targetId,
|
||||
detail: { name: target?.name ?? null },
|
||||
});
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -92,7 +92,7 @@ const updateSchema = z.object({
|
||||
.partial()
|
||||
.optional(),
|
||||
healthChecks: z
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365) })
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90) })
|
||||
.partial()
|
||||
.optional(),
|
||||
quietHours: z
|
||||
|
||||
@@ -18,6 +18,8 @@ export interface Alert {
|
||||
link: string | null;
|
||||
/** Under an active maintenance window: still a real problem, but notifications for it are held back. */
|
||||
silenced: boolean;
|
||||
/** What a maintenance window would have to be on to silence it, when there is such a thing — which is what "Silence" acts on. */
|
||||
target: { type: "server" | "integration" | "dns_provider"; id: number; name: string } | null;
|
||||
}
|
||||
|
||||
/** One integration that couldn't be read while collecting — so the page never mistakes "couldn't check" for "all clear". */
|
||||
|
||||
@@ -11,7 +11,8 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, secrets } from "../db/schema.js";
|
||||
import { integrations, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
|
||||
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
|
||||
@@ -95,8 +96,26 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const subjects = await activeSubjects();
|
||||
const intRows = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations);
|
||||
const intById = new Map(intRows.map((r) => [r.id, r]));
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((r) => [r.id, r.name]));
|
||||
|
||||
function push(a: { category: AlertCategory; severity: AlertSeverity; type: string; message: string; key: string; link?: string | null; silenced?: boolean }) {
|
||||
/** The thing a maintenance window would go on to silence an alert about `subject` ("server:3", "integration:1"). */
|
||||
const targetOf = (type: "server" | "integration" | "dns_provider", id: number): Alert["target"] => ({
|
||||
type,
|
||||
id,
|
||||
name: (type === "server" ? serverNames.get(id) : type === "integration" ? intById.get(id)?.name : undefined) ?? `${type} #${id}`,
|
||||
});
|
||||
const targetOfSubject = (subject: string | null): Alert["target"] => {
|
||||
const m = /^(server|integration|dns_provider):(\d+)$/.exec(subject ?? "");
|
||||
return m ? targetOf(m[1] as "server" | "integration" | "dns_provider", Number(m[2])) : null;
|
||||
};
|
||||
const integrationTarget = (id: number): Alert["target"] => targetOf("integration", id);
|
||||
/** Integration-down alerts are per kind of service; a window on any integration of that kind silences them, so offer the first. */
|
||||
const firstIntegrationOfType = (type: string): Alert["target"] => {
|
||||
const row = intRows.find((r) => r.type === type && r.enabled);
|
||||
return row ? integrationTarget(row.id) : null;
|
||||
};
|
||||
|
||||
function push(a: { category: AlertCategory; severity: AlertSeverity; type: string; message: string; key: string; link?: string | null; silenced?: boolean; target?: Alert["target"] }) {
|
||||
const meta = SOURCES[a.type];
|
||||
alerts.push({
|
||||
id: `${a.category}:${a.key}`,
|
||||
@@ -106,6 +125,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
message: a.message,
|
||||
link: a.link === undefined ? (meta?.link ?? null) : a.link,
|
||||
silenced: !!a.silenced,
|
||||
target: a.target ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -150,7 +170,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}` : undefined,
|
||||
silenced: subject !== null && subjects.has(subject),
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
|
||||
@@ -167,7 +187,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
for (const c of evaluateAutomation(items).conditions) {
|
||||
const origin = originOfConditionKey(c.key);
|
||||
const subject = subjectOfConditionKey(c.key);
|
||||
push({ category: "automation", severity: "warning", type: origin.type, message: c.message, key: c.key, silenced: subject !== null && subjects.has(subject) });
|
||||
push({ category: "automation", severity: "warning", type: origin.type, message: c.message, key: c.key, silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject) });
|
||||
}
|
||||
for (const h of held) {
|
||||
const m = /^(semaphore|gitea):(\d+)$/.exec(h);
|
||||
@@ -187,6 +207,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
message: `Latest backup on ${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} didn't succeed [${f.integrationName}]: ${f.status}`,
|
||||
key: `proxmox-backup:${f.integrationId}:${f.node}`,
|
||||
silenced: f.silenced,
|
||||
target: integrationTarget(f.integrationId),
|
||||
});
|
||||
}
|
||||
for (const u of uncovered) {
|
||||
@@ -197,6 +218,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
message: `${u.guestName} (#${u.vmid}) on ${u.node} isn't covered by any backup job [${u.integrationName}]`,
|
||||
key: `proxmox-uncovered:${u.integrationId}:${u.vmid}`,
|
||||
silenced: u.silenced,
|
||||
target: integrationTarget(u.integrationId),
|
||||
});
|
||||
}
|
||||
sourceFailures.forEach((f) => cannotCheckIntegration(f, "Proxmox backups"));
|
||||
@@ -214,6 +236,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
: `Datastore "${p.datastore}" has ${plural(p.failedCount, "snapshot")} that failed verification [${p.integrationName}]`,
|
||||
key: `pbs:${p.integrationId}:${p.datastore}`,
|
||||
silenced: p.silenced,
|
||||
target: integrationTarget(p.integrationId),
|
||||
});
|
||||
}
|
||||
sourceFailures.forEach((f) => cannotCheckIntegration(f, "Backup verification"));
|
||||
@@ -229,6 +252,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
message: `${u.containerName} [${u.environmentName}, ${u.integrationName}] has an image update available${u.newerVersion ? ` → ${u.newerVersion}` : ""}`,
|
||||
key: `docker:${u.integrationId}:${u.environmentName}:${u.containerName}`,
|
||||
silenced: silencedIntegration(u.integrationId),
|
||||
target: integrationTarget(u.integrationId),
|
||||
});
|
||||
}
|
||||
failures.forEach((f) => cannotCheckIntegration(f, "Image updates"));
|
||||
@@ -244,6 +268,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
message: k.daysLeft < 0 ? `Key for ${k.deviceLabel} [${k.integrationName}] has expired` : `Key for ${k.deviceLabel} [${k.integrationName}] expires in ${plural(k.daysLeft, "day")}`,
|
||||
key: `tailscale-key:${k.integrationId}:${k.deviceLabel}`,
|
||||
silenced: silencedIntegration(k.integrationId),
|
||||
target: integrationTarget(k.integrationId),
|
||||
});
|
||||
}
|
||||
failures.forEach((f) => cannotCheckIntegration(f, "Tailscale keys"));
|
||||
@@ -263,6 +288,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
message: `Monitor "${m.name}" is down${m.target ? ` (${m.target}${m.port ? `:${m.port}` : ""})` : ""} [${row.name}]`,
|
||||
key: `kuma:${row.id}:${m.id}`,
|
||||
silenced: silencedIntegration(row.id),
|
||||
target: integrationTarget(row.id),
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -285,6 +311,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
message: `${plural(overdue, "open ticket")} ${overdue === 1 ? "is" : "are"} overdue [${row.name}]`,
|
||||
key: `osticket:${row.id}`,
|
||||
silenced: silencedIntegration(row.id),
|
||||
target: integrationTarget(row.id),
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -329,6 +356,27 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
}
|
||||
}),
|
||||
|
||||
// Not a notification: a disk that is on course to fill is something to see coming, so it is shown here, from the server's own history.
|
||||
check("Disk forecasts", async () => {
|
||||
const { healthChecks } = await getSettings();
|
||||
if (!(healthChecks.diskForecastDays > 0)) return;
|
||||
for (const s of await db.select({ id: servers.id, name: servers.name }).from(servers)) {
|
||||
const series = await getServerMetrics(s.id, 14);
|
||||
for (const f of disksFillingSoon(series, healthChecks.diskForecastDays, healthChecks.diskUsagePercent)) {
|
||||
push({
|
||||
category: "disk",
|
||||
severity: f.daysToFull <= 3 ? "critical" : "warning",
|
||||
type: "server",
|
||||
message: `Disk ${f.mount} on ${s.name} is filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")} (${Math.round(f.latest)}% used now)`,
|
||||
key: `disk-forecast:${s.id}:${f.mount}`,
|
||||
link: `/servers/${s.id}`,
|
||||
silenced: subjects.has(`server:${s.id}`),
|
||||
target: targetOf("server", s.id),
|
||||
});
|
||||
}
|
||||
}
|
||||
}),
|
||||
|
||||
check("Integration failures", async () => {
|
||||
const { notifications } = await getSettings();
|
||||
for (const f of getFailingSources()) {
|
||||
@@ -340,6 +388,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
key: `failing:${f.source}`,
|
||||
link: null,
|
||||
silenced: await isSourceInMaintenance(f.source),
|
||||
target: firstIntegrationOfType(f.source),
|
||||
});
|
||||
}
|
||||
}),
|
||||
@@ -369,6 +418,11 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
}
|
||||
|
||||
let cache: { at: number; report: AlertsReport } | null = null;
|
||||
|
||||
/** Forget the stored result, so the next look is a fresh one — after something changes what it would say (a maintenance window starting or ending). */
|
||||
export function invalidateAlertsCache(): void {
|
||||
cache = null;
|
||||
}
|
||||
let inFlight: Promise<AlertsReport> | null = null;
|
||||
|
||||
/** The current alerts, reusing a recent result unless `force` asks for a fresh one (and even then not more than once every few seconds). */
|
||||
|
||||
@@ -86,6 +86,10 @@ export function bucketPoints(points: { t: number; v: number }[], from: number, t
|
||||
export interface Trend {
|
||||
/** Change in the percentage per day, from a straight line through the points. */
|
||||
perDay: number;
|
||||
/** How well a straight line describes the points, 0 to 1 (R²). A steady climb is near 1; a single jump or a wobble is low. */
|
||||
fit: number;
|
||||
/** How much time the points cover, in days. */
|
||||
spanDays: number;
|
||||
/** For something that fills up: days until it reaches 100% at that rate. Null when it isn't rising or the rate is negligible. */
|
||||
daysToFull: number | null;
|
||||
}
|
||||
@@ -106,16 +110,49 @@ export function trendOf(points: { t: number; v: number }[]): Trend | null {
|
||||
const meanY = ys.reduce((a, b) => a + b, 0) / n;
|
||||
let num = 0;
|
||||
let den = 0;
|
||||
let totalVariance = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
num += (xs[i] - meanX) * (ys[i] - meanY);
|
||||
den += (xs[i] - meanX) ** 2;
|
||||
totalVariance += (ys[i] - meanY) ** 2;
|
||||
}
|
||||
if (den === 0) return null;
|
||||
const slope = num / den;
|
||||
// R²: the share of the movement the straight line explains. A flat series has nothing to explain, so it is no "fit" at all.
|
||||
const fit = totalVariance === 0 ? 0 : Math.round(((num * num) / (den * totalVariance)) * 100) / 100;
|
||||
const perDay = Math.round(slope * 100) / 100;
|
||||
const last = ys[n - 1];
|
||||
const daysToFull = perDay >= 0.05 && last < 100 ? Math.ceil((100 - last) / perDay) : null;
|
||||
return { perDay, daysToFull };
|
||||
return { perDay, fit, spanDays: Math.round((span / DAY_MS) * 10) / 10, daysToFull };
|
||||
}
|
||||
|
||||
export interface DiskForecast {
|
||||
mount: string;
|
||||
latest: number;
|
||||
perDay: number;
|
||||
daysToFull: number;
|
||||
}
|
||||
|
||||
/** What a forecast needs before it is worth raising an alert about: enough history, and growth that really is steady. */
|
||||
export const FORECAST_MIN_SPAN_DAYS = 3;
|
||||
export const FORECAST_MIN_FIT = 0.9;
|
||||
|
||||
/**
|
||||
* The disks that are heading for full within `withinDays`. A disk already at or above `alreadyAlertedAt` percent is left out — the
|
||||
* ordinary "disk nearly full" alert covers it — and so is one whose growth is erratic (a big copy, a log that rotates), where a
|
||||
* straight-line forecast would just be a guess.
|
||||
*/
|
||||
export function disksFillingSoon(series: MetricSeries[], withinDays: number, alreadyAlertedAt: number): DiskForecast[] {
|
||||
const out: DiskForecast[] = [];
|
||||
for (const s of series) {
|
||||
if (s.kind !== "disk" || !s.trend || s.latest === null) continue;
|
||||
const { perDay, fit, spanDays, daysToFull } = s.trend;
|
||||
if (daysToFull === null || daysToFull > withinDays) continue;
|
||||
if (spanDays < FORECAST_MIN_SPAN_DAYS || fit < FORECAST_MIN_FIT) continue;
|
||||
if (s.latest >= alreadyAlertedAt) continue;
|
||||
out.push({ mount: s.metric.slice("disk:".length), latest: s.latest, perDay, daysToFull });
|
||||
}
|
||||
return out.sort((a, b) => a.daysToFull - b.daysToFull);
|
||||
}
|
||||
|
||||
export interface MetricSeries {
|
||||
|
||||
@@ -95,6 +95,8 @@ export interface HealthCheckSettings {
|
||||
diskUsagePercent: number;
|
||||
/** A domain registration expiring within this many days is reminded about daily (same schedule as secret expiry). */
|
||||
domainWarnDays: number;
|
||||
/** An agent-reported disk that is on course to be full within this many days is flagged on the Alerts page. 0 turns it off. */
|
||||
diskForecastDays: number;
|
||||
}
|
||||
|
||||
export interface ConsistencySettings {
|
||||
@@ -180,7 +182,7 @@ const DEFAULTS: AppSettings = {
|
||||
display: { dateFormat: "ymd", timeFormat: "24h", pageSize: 20 },
|
||||
logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 },
|
||||
quietHours: { enabled: false, start: "22:00", end: "07:00" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30 },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
consistency: { excludedRanges: ["172.16.0.0/12"] },
|
||||
|
||||
Reference in new issue
Block a user