Flapping alerts, a noise report and bulk actions with undo
Flapping: an alert raised three times within an hour is announced once as flapping, its further raise/clear messages are held until it has been calm for an hour, and one message then says whether it is still active. Event webhooks still hear everything. The state survives restarts. Alerts > Noise lists the alerts that keep coming back (7-180 days): how often, how long, most in an hour, and a hint on what helps. Bulk actions on Alerts (acknowledge, remove acknowledgement, silence), Servers (tag, silence, runs on) and Inventory (location and dates, print labels). Each reports what it left alone and why, and has an Undo on the toast. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
be6c5552bd
commit
73a06ebe63
35 files changed
+2546
-17
No files matched your search
@@ -7,6 +7,7 @@ import { db } from "../db/client.js";
|
||||
import { inArray } from "drizzle-orm";
|
||||
import { pushSubscriptions, servers } from "../db/schema.js";
|
||||
import { parseTags } from "./serverTags.js";
|
||||
import { dampRaised, dampRecovered } from "./flapGuard.js";
|
||||
import { firstMatchingRule, metaOfConditionKey, pushAccepts, type ChannelId, type NotifyMeta } from "./notifyRouting.js";
|
||||
|
||||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||||
@@ -483,7 +484,12 @@ export async function notifyHealthIssues(issues: { message: string; key?: string
|
||||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||||
if (issues.length === 0) return;
|
||||
// An alert that keeps coming and going is announced once as "flapping" and then left quiet (services/flapGuard.ts); that is tracked whether or not this is switched on.
|
||||
const { pass, started } = await dampRaised(issues);
|
||||
if (!(await eventEnabled("healthAlerts"))) return;
|
||||
if (started.length > 0) await notifyFlapping(started);
|
||||
if (pass.length === 0) return;
|
||||
issues = pass;
|
||||
await notifyGrouped(issues, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Health Alert",
|
||||
message: `${group.length} new issue${group.length !== 1 ? "s" : ""}:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`,
|
||||
@@ -493,13 +499,34 @@ export async function notifyHealthIssues(issues: { message: string; key?: string
|
||||
export async function notifyHealthRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||||
if (resolved.length === 0) return;
|
||||
const passed = await dampRecovered(resolved);
|
||||
if (!(await eventEnabled("healthAlerts"))) return;
|
||||
if (passed.length === 0) return;
|
||||
resolved = passed;
|
||||
await notifyGrouped(resolved, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Health Recovered",
|
||||
message: `${group.length} issue${group.length !== 1 ? "s" : ""} cleared:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||||
}));
|
||||
}
|
||||
|
||||
/** An alert that has just been called flapping: said once, with how often it has come, instead of each time. */
|
||||
export async function notifyFlapping(items: { message: string; key?: string; severity?: string; count: number }[]): Promise<void> {
|
||||
if (items.length === 0) return;
|
||||
await notifyGrouped(items, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Flapping alert",
|
||||
message: `${group.length === 1 ? "An alert keeps" : `${group.length} alerts keep`} coming and going:\n\n${group.map((i) => `↻ ${i.message} — raised ${i.count} times within an hour`).join("\n")}\n\nIts further messages are held back until it has been quiet for an hour; it stays on the Alerts page, marked as flapping.`,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Flapping alerts that have been quiet for an hour: what state each is in now. */
|
||||
export async function notifySettled(items: { message: string; key: string; active: boolean }[]): Promise<void> {
|
||||
if (items.length === 0 || !(await eventEnabled("healthAlerts"))) return;
|
||||
await notifyGrouped(items, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Alert settled",
|
||||
message: `${group.length === 1 ? "An alert that was flapping has" : `${group.length} alerts that were flapping have`} been steady for an hour:\n\n${group.map((i) => `${i.active ? "⚠ still active" : "✓ cleared"} — ${i.message}`).join("\n")}`,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
|
||||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||||
|
||||
Reference in new issue
Block a user