Disk fill forecasts, skeleton loaders, and a Silence button on alerts

A disk that is on course to be full within 14 days (setting: 0 turns it off) is
flagged on the Alerts page and dashboard, from the server's own history: needs
three days of steady growth, skips erratic disks and ones the ordinary disk
alert covers. Skeleton placeholders replace "Fetching status…" and "Loading…".
Alerts that a maintenance window can silence get a Silence button (1, 4 or 24
hours) that starts one, and the alert list is refreshed at once when a window
starts or ends.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-15 00:12:33 +02:00
1 parent 82974b98d0
commit f7f5e1cd41
24 files changed
+566 -25

No files matched your search

+58 -4
View File
@@ -11,7 +11,8 @@
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, secrets } from "../db/schema.js";
import { integrations, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
@@ -95,8 +96,26 @@ export async function collectAlerts(): Promise<AlertsReport> {
const subjects = await activeSubjects();
const intRows = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations);
const intById = new Map(intRows.map((r) => [r.id, r]));
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((r) => [r.id, r.name]));
function push(a: { category: AlertCategory; severity: AlertSeverity; type: string; message: string; key: string; link?: string | null; silenced?: boolean }) {
/** The thing a maintenance window would go on to silence an alert about `subject` ("server:3", "integration:1"). */
const targetOf = (type: "server" | "integration" | "dns_provider", id: number): Alert["target"] => ({
type,
id,
name: (type === "server" ? serverNames.get(id) : type === "integration" ? intById.get(id)?.name : undefined) ?? `${type} #${id}`,
});
const targetOfSubject = (subject: string | null): Alert["target"] => {
const m = /^(server|integration|dns_provider):(\d+)$/.exec(subject ?? "");
return m ? targetOf(m[1] as "server" | "integration" | "dns_provider", Number(m[2])) : null;
};
const integrationTarget = (id: number): Alert["target"] => targetOf("integration", id);
/** Integration-down alerts are per kind of service; a window on any integration of that kind silences them, so offer the first. */
const firstIntegrationOfType = (type: string): Alert["target"] => {
const row = intRows.find((r) => r.type === type && r.enabled);
return row ? integrationTarget(row.id) : null;
};
function push(a: { category: AlertCategory; severity: AlertSeverity; type: string; message: string; key: string; link?: string | null; silenced?: boolean; target?: Alert["target"] }) {
const meta = SOURCES[a.type];
alerts.push({
id: `${a.category}:${a.key}`,
@@ -106,6 +125,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
message: a.message,
link: a.link === undefined ? (meta?.link ?? null) : a.link,
silenced: !!a.silenced,
target: a.target ?? null,
});
}
@@ -150,7 +170,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}` : undefined,
silenced: subject !== null && subjects.has(subject),
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
@@ -167,7 +187,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
for (const c of evaluateAutomation(items).conditions) {
const origin = originOfConditionKey(c.key);
const subject = subjectOfConditionKey(c.key);
push({ category: "automation", severity: "warning", type: origin.type, message: c.message, key: c.key, silenced: subject !== null && subjects.has(subject) });
push({ category: "automation", severity: "warning", type: origin.type, message: c.message, key: c.key, silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject) });
}
for (const h of held) {
const m = /^(semaphore|gitea):(\d+)$/.exec(h);
@@ -187,6 +207,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
message: `Latest backup on ${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} didn't succeed [${f.integrationName}]: ${f.status}`,
key: `proxmox-backup:${f.integrationId}:${f.node}`,
silenced: f.silenced,
target: integrationTarget(f.integrationId),
});
}
for (const u of uncovered) {
@@ -197,6 +218,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
message: `${u.guestName} (#${u.vmid}) on ${u.node} isn't covered by any backup job [${u.integrationName}]`,
key: `proxmox-uncovered:${u.integrationId}:${u.vmid}`,
silenced: u.silenced,
target: integrationTarget(u.integrationId),
});
}
sourceFailures.forEach((f) => cannotCheckIntegration(f, "Proxmox backups"));
@@ -214,6 +236,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
: `Datastore "${p.datastore}" has ${plural(p.failedCount, "snapshot")} that failed verification [${p.integrationName}]`,
key: `pbs:${p.integrationId}:${p.datastore}`,
silenced: p.silenced,
target: integrationTarget(p.integrationId),
});
}
sourceFailures.forEach((f) => cannotCheckIntegration(f, "Backup verification"));
@@ -229,6 +252,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
message: `${u.containerName} [${u.environmentName}, ${u.integrationName}] has an image update available${u.newerVersion ? ` → ${u.newerVersion}` : ""}`,
key: `docker:${u.integrationId}:${u.environmentName}:${u.containerName}`,
silenced: silencedIntegration(u.integrationId),
target: integrationTarget(u.integrationId),
});
}
failures.forEach((f) => cannotCheckIntegration(f, "Image updates"));
@@ -244,6 +268,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
message: k.daysLeft < 0 ? `Key for ${k.deviceLabel} [${k.integrationName}] has expired` : `Key for ${k.deviceLabel} [${k.integrationName}] expires in ${plural(k.daysLeft, "day")}`,
key: `tailscale-key:${k.integrationId}:${k.deviceLabel}`,
silenced: silencedIntegration(k.integrationId),
target: integrationTarget(k.integrationId),
});
}
failures.forEach((f) => cannotCheckIntegration(f, "Tailscale keys"));
@@ -263,6 +288,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
message: `Monitor "${m.name}" is down${m.target ? ` (${m.target}${m.port ? `:${m.port}` : ""})` : ""} [${row.name}]`,
key: `kuma:${row.id}:${m.id}`,
silenced: silencedIntegration(row.id),
target: integrationTarget(row.id),
});
}
} catch (err) {
@@ -285,6 +311,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
message: `${plural(overdue, "open ticket")} ${overdue === 1 ? "is" : "are"} overdue [${row.name}]`,
key: `osticket:${row.id}`,
silenced: silencedIntegration(row.id),
target: integrationTarget(row.id),
});
}
} catch (err) {
@@ -329,6 +356,27 @@ export async function collectAlerts(): Promise<AlertsReport> {
}
}),
// Not a notification: a disk that is on course to fill is something to see coming, so it is shown here, from the server's own history.
check("Disk forecasts", async () => {
const { healthChecks } = await getSettings();
if (!(healthChecks.diskForecastDays > 0)) return;
for (const s of await db.select({ id: servers.id, name: servers.name }).from(servers)) {
const series = await getServerMetrics(s.id, 14);
for (const f of disksFillingSoon(series, healthChecks.diskForecastDays, healthChecks.diskUsagePercent)) {
push({
category: "disk",
severity: f.daysToFull <= 3 ? "critical" : "warning",
type: "server",
message: `Disk ${f.mount} on ${s.name} is filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")} (${Math.round(f.latest)}% used now)`,
key: `disk-forecast:${s.id}:${f.mount}`,
link: `/servers/${s.id}`,
silenced: subjects.has(`server:${s.id}`),
target: targetOf("server", s.id),
});
}
}
}),
check("Integration failures", async () => {
const { notifications } = await getSettings();
for (const f of getFailingSources()) {
@@ -340,6 +388,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
key: `failing:${f.source}`,
link: null,
silenced: await isSourceInMaintenance(f.source),
target: firstIntegrationOfType(f.source),
});
}
}),
@@ -369,6 +418,11 @@ export async function collectAlerts(): Promise<AlertsReport> {
}
let cache: { at: number; report: AlertsReport } | null = null;
/** Forget the stored result, so the next look is a fresh one — after something changes what it would say (a maintenance window starting or ending). */
export function invalidateAlertsCache(): void {
cache = null;
}
let inFlight: Promise<AlertsReport> | null = null;
/** The current alerts, reusing a recent result unless `force` asks for a fresh one (and even then not more than once every few seconds). */