Alert rules of your own, storage forecasts for Proxmox and Synology, and snooze

- Monitoring -> Alert rules: CPU, memory, disk use and disk temperature limits with a time, severity and scope (all, one server, a tag); try before saving; findings are ordinary health conditions
- Proxmox storages and Synology volumes are written down on each health pass (hourly, 400 days) and forecast like server disks: badge and history chart on the pages, alert, digest line, /metrics gauge
- Snooze an alert until a time; held back like maintenance, messages held, announced once again if still there when it ends
- Docs updated

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-20 23:38:36 +02:00
1 parent 788b3e073c
commit 24ae8db380
38 files changed
+7847 -8

No files matched your search

+19 -2
View File
@@ -8,6 +8,7 @@ import { inArray } from "drizzle-orm";
import { pushSubscriptions, servers } from "../db/schema.js";
import { parseTags } from "./serverTags.js";
import { dampRaised, dampRecovered } from "./flapGuard.js";
import { withoutSnoozed } from "./alertSnoozes.js";
import { firstMatchingRule, metaOfConditionKey, pushAccepts, type ChannelId, type NotifyMeta } from "./notifyRouting.js";
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
@@ -484,6 +485,9 @@ export async function notifyHealthIssues(issues: { message: string; key?: string
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
// One somebody has snoozed is not announced now; if it is still there when the snooze ends, it is announced then (services/alertSnoozes.ts).
issues = await withoutSnoozed(issues);
if (issues.length === 0) return;
// An alert that keeps coming and going is announced once as "flapping" and then left quiet (services/flapGuard.ts); that is tracked whether or not this is switched on.
const { pass, started } = await dampRaised(issues);
if (!(await eventEnabled("healthAlerts"))) return;
@@ -499,6 +503,8 @@ export async function notifyHealthIssues(issues: { message: string; key?: string
export async function notifyHealthRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
resolved = await withoutSnoozed(resolved);
if (resolved.length === 0) return;
const passed = await dampRecovered(resolved);
if (!(await eventEnabled("healthAlerts"))) return;
if (passed.length === 0) return;
@@ -509,6 +515,15 @@ export async function notifyHealthRecovered(resolved: { message: string; key?: s
}));
}
/** Alerts that were snoozed, whose time is up and that are still there: said once, as a reminder. */
export async function notifySnoozeEnded(items: { message: string; key?: string; severity?: string }[]): Promise<void> {
if (items.length === 0) return;
await notifyGrouped(items, issueMeta, (group) => ({
title: "Homelab Manager — Snooze ended",
message: `${group.length === 1 ? "An alert you snoozed is" : `${group.length} alerts you snoozed are`} still there:\n\n${group.map((i) => `⏰ ${i.message}`).join("\n")}`,
}));
}
/** An alert that has just been called flapping: said once, with how often it has come, instead of each time. */
export async function notifyFlapping(items: { message: string; key?: string; severity?: string; count: number }[]): Promise<void> {
if (items.length === 0) return;
@@ -531,7 +546,8 @@ export async function notifyAutomationFailed(issues: { message: string; key?: st
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
if (!(await eventEnabled("automationAlerts"))) return;
issues = await withoutSnoozed(issues);
if (issues.length === 0 || !(await eventEnabled("automationAlerts"))) return;
await notifyGrouped(issues, issueMeta, (group) => ({
title: "Homelab Manager — Automation Failed",
message: `${group.length} run${group.length !== 1 ? "s" : ""} failed:\n\n${group.map((i) => `✕ ${i.message}`).join("\n")}`,
@@ -541,7 +557,8 @@ export async function notifyAutomationFailed(issues: { message: string; key?: st
export async function notifyAutomationRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
if (!(await eventEnabled("automationAlerts"))) return;
resolved = await withoutSnoozed(resolved);
if (resolved.length === 0 || !(await eventEnabled("automationAlerts"))) return;
await notifyGrouped(resolved, issueMeta, (group) => ({
title: "Homelab Manager — Automation Recovered",
message: `${group.length} no longer failing:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,