Alert rules of your own, storage forecasts for Proxmox and Synology, and snooze
- Monitoring -> Alert rules: CPU, memory, disk use and disk temperature limits with a time, severity and scope (all, one server, a tag); try before saving; findings are ordinary health conditions - Proxmox storages and Synology volumes are written down on each health pass (hourly, 400 days) and forecast like server disks: badge and history chart on the pages, alert, digest line, /metrics gauge - Snooze an alert until a time; held back like maintenance, messages held, announced once again if still there when it ends - Docs updated Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
788b3e073c
commit
24ae8db380
38 files changed
+7847
-8
No files matched your search
@@ -8,6 +8,7 @@ import { inArray } from "drizzle-orm";
|
||||
import { pushSubscriptions, servers } from "../db/schema.js";
|
||||
import { parseTags } from "./serverTags.js";
|
||||
import { dampRaised, dampRecovered } from "./flapGuard.js";
|
||||
import { withoutSnoozed } from "./alertSnoozes.js";
|
||||
import { firstMatchingRule, metaOfConditionKey, pushAccepts, type ChannelId, type NotifyMeta } from "./notifyRouting.js";
|
||||
|
||||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||||
@@ -484,6 +485,9 @@ export async function notifyHealthIssues(issues: { message: string; key?: string
|
||||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||||
if (issues.length === 0) return;
|
||||
// One somebody has snoozed is not announced now; if it is still there when the snooze ends, it is announced then (services/alertSnoozes.ts).
|
||||
issues = await withoutSnoozed(issues);
|
||||
if (issues.length === 0) return;
|
||||
// An alert that keeps coming and going is announced once as "flapping" and then left quiet (services/flapGuard.ts); that is tracked whether or not this is switched on.
|
||||
const { pass, started } = await dampRaised(issues);
|
||||
if (!(await eventEnabled("healthAlerts"))) return;
|
||||
@@ -499,6 +503,8 @@ export async function notifyHealthIssues(issues: { message: string; key?: string
|
||||
export async function notifyHealthRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||||
if (resolved.length === 0) return;
|
||||
resolved = await withoutSnoozed(resolved);
|
||||
if (resolved.length === 0) return;
|
||||
const passed = await dampRecovered(resolved);
|
||||
if (!(await eventEnabled("healthAlerts"))) return;
|
||||
if (passed.length === 0) return;
|
||||
@@ -509,6 +515,15 @@ export async function notifyHealthRecovered(resolved: { message: string; key?: s
|
||||
}));
|
||||
}
|
||||
|
||||
/** Alerts that were snoozed, whose time is up and that are still there: said once, as a reminder. */
|
||||
export async function notifySnoozeEnded(items: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||||
if (items.length === 0) return;
|
||||
await notifyGrouped(items, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Snooze ended",
|
||||
message: `${group.length === 1 ? "An alert you snoozed is" : `${group.length} alerts you snoozed are`} still there:\n\n${group.map((i) => `⏰ ${i.message}`).join("\n")}`,
|
||||
}));
|
||||
}
|
||||
|
||||
/** An alert that has just been called flapping: said once, with how often it has come, instead of each time. */
|
||||
export async function notifyFlapping(items: { message: string; key?: string; severity?: string; count: number }[]): Promise<void> {
|
||||
if (items.length === 0) return;
|
||||
@@ -531,7 +546,8 @@ export async function notifyAutomationFailed(issues: { message: string; key?: st
|
||||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||||
if (issues.length === 0) return;
|
||||
if (!(await eventEnabled("automationAlerts"))) return;
|
||||
issues = await withoutSnoozed(issues);
|
||||
if (issues.length === 0 || !(await eventEnabled("automationAlerts"))) return;
|
||||
await notifyGrouped(issues, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Automation Failed",
|
||||
message: `${group.length} run${group.length !== 1 ? "s" : ""} failed:\n\n${group.map((i) => `✕ ${i.message}`).join("\n")}`,
|
||||
@@ -541,7 +557,8 @@ export async function notifyAutomationFailed(issues: { message: string; key?: st
|
||||
export async function notifyAutomationRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||||
if (resolved.length === 0) return;
|
||||
if (!(await eventEnabled("automationAlerts"))) return;
|
||||
resolved = await withoutSnoozed(resolved);
|
||||
if (resolved.length === 0 || !(await eventEnabled("automationAlerts"))) return;
|
||||
await notifyGrouped(resolved, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Automation Recovered",
|
||||
message: `${group.length} no longer failing:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||||
|
||||
Reference in new issue
Block a user