Alert rules of your own, storage forecasts for Proxmox and Synology, and snooze

- Monitoring -> Alert rules: CPU, memory, disk use and disk temperature limits with a time, severity and scope (all, one server, a tag); try before saving; findings are ordinary health conditions
- Proxmox storages and Synology volumes are written down on each health pass (hourly, 400 days) and forecast like server disks: badge and history chart on the pages, alert, digest line, /metrics gauge
- Snooze an alert until a time; held back like maintenance, messages held, announced once again if still there when it ends
- Docs updated

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-20 23:38:36 +02:00
1 parent 788b3e073c
commit 24ae8db380
38 files changed
+7847 -8

No files matched your search

+39 -2
View File
@@ -13,6 +13,7 @@ import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, haEntities, haWatched, heartbeats, integrations, networkDevices, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
import { storageFillingSoon, storageOverview } from "./storageHistory.js";
import { daysOfSupportLeft, lifecycleOf } from "./osLifecycle.js";
import { npmIntegrationIds, npmProblems, readNpm } from "./npmReader.js";
import { haProblems, haSyncStatuses, isHaFresh } from "./homeAssistantSync.js";
@@ -21,6 +22,7 @@ import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
import { recordFromReport } from "./incidents.js";
import { reconcileAcks } from "./alertAcks.js";
import { flappingKeys, keyOfAlertId } from "./flapGuard.js";
import { loadSnoozes } from "./alertSnoozes.js";
import { lastAttempt as lastSelfBackup, newestSnapshot, STALE_AFTER_HOURS as SELF_BACKUP_STALE_HOURS } from "./selfBackup.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
@@ -88,7 +90,9 @@ const SEVERITY_ORDER: Record<AlertSeverity, number> = { critical: 0, warning: 1,
function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } {
const device = /^offline:device:(\d+)$/.exec(key);
if (device) return { type: "unifi", id: Number(device[1]), category: "offline" };
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
let m = /^rule:\d+:server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: "monitoring" };
m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : key.startsWith("new-port") ? "monitoring" : "disk" };
m = /^disk:proxmox:(\d+):/.exec(key);
if (m) return { type: "proxmox", id: Number(m[1]), category: "disk" };
@@ -217,11 +221,12 @@ export async function collectAlerts(): Promise<AlertsReport> {
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : ""}` : origin.type === "unifi" ? "/network/devices" : undefined,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
if (held.has("server:rules")) cannotCheck("Alert rules", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
@@ -520,6 +525,26 @@ export async function collectAlerts(): Promise<AlertsReport> {
}
}),
// The same for the pools the integrations report (a Proxmox storage, a Synology volume), from the history kept on each health pass.
check("Storage forecasts", async () => {
const { healthChecks } = await getSettings();
if (!(healthChecks.diskForecastDays > 0)) return;
const nowMs = Date.now();
const rows = (await storageOverview()).filter((r) => nowMs - Date.parse(r.updatedAt) < 2 * 86_400_000 && intById.get(r.integrationId)?.enabled);
for (const f of storageFillingSoon(rows, healthChecks.diskForecastDays, healthChecks.diskUsagePercent)) {
const row = intById.get(f.integrationId)!;
push({
category: "disk",
severity: f.daysToFull <= 3 ? "critical" : "warning",
type: row.type,
message: `${f.label} is filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")} (${Math.round(f.latest)}% used now)`,
key: `storage-forecast:${f.integrationId}:${f.key}`,
silenced: silencedIntegration(f.integrationId),
target: integrationTarget(f.integrationId),
});
}
}),
// Certificates in use by Nginx Proxy Manager that are running out, and hosts it could not set up.
check("Nginx Proxy Manager", async () => {
for (const i of await npmIntegrationIds()) {
@@ -617,6 +642,18 @@ export async function collectAlerts(): Promise<AlertsReport> {
const flapping = await flappingKeys().catch(() => new Set<string>());
if (flapping.size > 0) for (const a of alerts) if (flapping.has(keyOfAlertId(a.id, a.category))) a.flapping = true;
// Those somebody has snoozed are held back like those under maintenance, with when they come back.
const snoozes = await loadSnoozes().catch(() => new Map<string, { until: string; snoozedBy: string }>());
if (snoozes.size > 0) {
for (const a of alerts) {
const sn = snoozes.get(a.id);
if (sn) {
a.snoozed = { until: sn.until, by: sn.snoozedBy };
a.silenced = true;
}
}
}
const active = alerts.filter((a) => !a.silenced);
const grouped = alerts.filter((a) => a.causedBy).length;
return {