Flapping alerts, a noise report and bulk actions with undo

Flapping: an alert raised three times within an hour is announced once as
flapping, its further raise/clear messages are held until it has been calm
for an hour, and one message then says whether it is still active. Event
webhooks still hear everything. The state survives restarts.

Alerts > Noise lists the alerts that keep coming back (7-180 days): how often,
how long, most in an hour, and a hint on what helps.

Bulk actions on Alerts (acknowledge, remove acknowledgement, silence), Servers
(tag, silence, runs on) and Inventory (location and dates, print labels). Each
reports what it left alone and why, and has an Undo on the toast.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 23:55:09 +02:00
1 parent be6c5552bd
commit 73a06ebe63
35 files changed
+2546 -17

No files matched your search

+5
View File
@@ -52,6 +52,7 @@ import { heartbeatsRouter } from "./routes/heartbeats.js";
import { agendaRouter } from "./routes/agenda.js";
import { patchPlansRouter } from "./routes/patchPlans.js";
import { reportsRouter } from "./routes/reports.js";
import { alertsBulkRouter, inventoryBulkRouter, serversBulkRouter } from "./routes/bulk.js";
import { publicCalendarRouter } from "./routes/publicCalendar.js";
import { heartbeatPingRouter } from "./routes/heartbeatPing.js";
import { apiErrorHandler } from "./utils/apiErrorHandler.js";
@@ -123,6 +124,8 @@ export function createApp() {
app.use("/api/update-runs", updateRunsRouter);
app.use("/api/db-backups", dbBackupsRouter);
app.use("/api/incidents", incidentsRouter);
// The bulk routes come before the routers of the same things, so "/bulk" is not read as an id.
app.use("/api/inventory/bulk", inventoryBulkRouter);
app.use("/api/inventory", inventoryRouter);
app.use("/api/dependencies", dependenciesRouter);
app.use("/api/push", pushRouter);
@@ -142,6 +145,7 @@ export function createApp() {
app.use("/api/secrets", secretsRouter);
app.use("/api/ipam", ipamRouter);
app.use("/api/dns", dnsRouter);
app.use("/api/servers/bulk", serversBulkRouter);
app.use("/api/servers", serversRouter);
app.use("/api/tasks", tasksRouter);
app.use("/api/integrations", integrationsRouter);
@@ -154,6 +158,7 @@ export function createApp() {
app.use("/api/privacy", privacyRouter);
app.use("/api/tags", tagsRouter);
app.use("/api/ports", portsRouter);
app.use("/api/alerts/bulk", alertsBulkRouter);
app.use("/api/alerts", alertsRouter);
app.use("/api/map", mapRouter);
app.use("/api/generator", generatorRouter);
+2
View File
@@ -23,6 +23,7 @@ import { initUpdateRuns } from "./services/updateRuns.js";
import { initEscalation } from "./services/escalation.js";
import { initEndpointMonitor } from "./services/endpointMonitor.js";
import { initPatchPlans } from "./services/patchPlans.js";
import { initFlapMonitor } from "./services/flapMonitor.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
import { initIncidentRecorder } from "./services/incidents.js";
import { getAlerts } from "./services/alerts.js";
@@ -65,6 +66,7 @@ initIncidentRecorder(() => getAlerts(false));
initEscalation();
initEndpointMonitor();
initPatchPlans();
initFlapMonitor();
createApp().listen(env.port, () => {
console.log(`homelab-manager listening on port ${env.port}`);
+7
View File
@@ -5,6 +5,7 @@ import { getAlerts } from "../services/alerts.js";
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
import { recordAudit } from "../services/audit.js";
import { openIncidentIds } from "../services/incidents.js";
import { buildNoiseReport, NOISE_DAYS } from "../services/alertNoise.js";
import { ticketsForAlerts, type AlertTicketInfo } from "../services/alertTickets.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -41,6 +42,12 @@ alertsRouter.get("/", asyncHandler(async (req, res) => {
});
}));
// Which alerts fire most often and how long they last, from the incident record — for everyone signed in, like the history it comes from. ?days=7|30|90|180.
alertsRouter.get("/noise", asyncHandler(async (req, res) => {
const asked = Number(req.query.days);
res.json(await buildNoiseReport((NOISE_DAYS as readonly number[]).includes(asked) ? asked : 30));
}));
// Who an alert can be handed to.
alertsRouter.get("/assignees", requireRole("operator"), asyncHandler(async (_req, res) => {
res.json({ users: await assignableUsers() });
+184
View File
@@ -0,0 +1,184 @@
import { Router } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { AckError, MAX_HOURS, MAX_NOTE } from "../services/alertAcks.js";
import { recordAudit } from "../services/audit.js";
import { acknowledgeAlerts, BulkInputError, changeAssets, changeServerTag, MAX_BULK, restoreAcks, restoreAssets, restoreServerParents, setServerParents, setServerTags, silenceTargets, targetsOfAlerts, undoSilence, unacknowledgeAlerts } from "../services/bulkActions.js";
import { InvalidTagError } from "../services/serverTags.js";
import { asyncHandler } from "../utils/asyncHandler.js";
// Bulk actions are the single-item ones done to several at once, so each needs the same role the single one does; and every one returns the body that puts it back,
// which is all an "Undo" sends. One audit entry says what was done to how many.
const intIds = z.array(z.number().int().positive()).min(1).max(MAX_BULK);
const stringIds = z.array(z.string().min(1).max(600)).min(1).max(MAX_BULK);
const minutes = z.number().int().min(5).max(7 * 24 * 60);
const reason = z.string().trim().max(200).optional();
const detailIds = (ids: unknown[]) => ids.slice(0, 50);
const bad = (res: import("express").Response, message: string) => res.status(400).json({ error: "invalid_body", message });
const nameOf = (req: import("express").Request) => {
const u = req.currentUser!;
return u.name ?? u.email ?? u.oidcSub;
};
// ─── Servers ───────────────────────────────────────────────────────────────
export const serversBulkRouter = Router();
serversBulkRouter.use(requireAuth);
const serversBody = z.discriminatedUnion("action", [
z.object({ action: z.literal("add_tag"), ids: intIds, tag: z.string().min(1).max(100) }),
z.object({ action: z.literal("remove_tag"), ids: intIds, tag: z.string().min(1).max(100) }),
z.object({ action: z.literal("set_tags"), items: z.array(z.object({ id: z.number().int().positive(), tags: z.array(z.string().max(100)).max(50) })).min(1).max(MAX_BULK) }),
z.object({ action: z.literal("silence"), ids: intIds, minutes, reason, withDependents: z.boolean().optional() }),
z.object({ action: z.literal("undo_silence"), created: z.array(z.number().int().positive()).max(MAX_BULK * 4), restored: z.array(z.object({ id: z.number().int().positive(), endsAt: z.string().max(40), reason: z.string().max(200).nullable() })).max(MAX_BULK * 4) }),
z.object({ action: z.literal("set_parent"), ids: intIds, parentServerId: z.number().int().positive().nullable() }),
z.object({ action: z.literal("restore_parents"), items: z.array(z.object({ id: z.number().int().positive(), parentServerId: z.number().int().positive().nullable(), noParent: z.boolean() })).min(1).max(MAX_BULK) }),
]);
serversBulkRouter.post("/", asyncHandler(async (req, res) => {
const parsed = serversBody.safeParse(req.body);
if (!parsed.success) return bad(res, parsed.error.issues[0]?.message ?? `Choose up to ${MAX_BULK} servers and what to do.`);
const body = parsed.data;
const actor = req.currentUser!;
// Tags and maintenance are an operator's, like one server's; which server another runs on is an administrator's.
const needsAdmin = body.action === "set_parent" || body.action === "restore_parents";
if (actor.role !== "admin" && (needsAdmin || actor.role !== "operator")) return res.status(403).json({ error: "forbidden", message: needsAdmin ? "Changing which server another runs on needs an administrator." : "That needs an operator." });
try {
switch (body.action) {
case "add_tag":
case "remove_tag": {
const r = await changeServerTag(body.ids, body.action === "add_tag" ? { add: body.tag } : { remove: body.tag });
await recordAudit({ actor, category: "server", action: "bulk_tag", detail: { change: body.action, tag: body.tag, changed: r.changed, ids: detailIds(body.ids) } });
return res.json(r);
}
case "set_tags": {
const changed = await setServerTags(body.items);
await recordAudit({ actor, category: "server", action: "bulk_tag", detail: { change: "undo", changed, ids: detailIds(body.items.map((i) => i.id)) } });
return res.json({ changed, skipped: [], undo: null });
}
case "silence": {
const r = await silenceTargets(body.ids.map((id) => ({ type: "server" as const, id })), body.minutes, body.reason || "Maintenance on several servers", nameOf(req), body.withDependents === true);
await recordAudit({ actor, category: "maintenance", action: "bulk_start", detail: { servers: r.silenced.map((s) => s.name).slice(0, 50), minutes: body.minutes, reason: body.reason ?? null, withDependents: body.withDependents === true } });
return res.json({ changed: r.silenced.length, skipped: r.skipped, undo: r.undo, silenced: r.silenced });
}
case "undo_silence": {
const changed = await undoSilence(body);
await recordAudit({ actor, category: "maintenance", action: "bulk_undo", detail: { windows: changed } });
return res.json({ changed, skipped: [], undo: null });
}
case "set_parent": {
const r = await setServerParents(body.ids, body.parentServerId);
await recordAudit({ actor, category: "server", action: "bulk_runs_on", detail: { parentServerId: body.parentServerId, changed: r.changed, ids: detailIds(body.ids) } });
return res.json(r);
}
case "restore_parents": {
const changed = await restoreServerParents(body.items);
await recordAudit({ actor, category: "server", action: "bulk_runs_on", detail: { change: "undo", changed } });
return res.json({ changed, skipped: [], undo: null });
}
}
} catch (err) {
if (err instanceof InvalidTagError) return res.status(400).json({ error: "invalid_tag", message: err.message });
if (err instanceof RangeError) return res.status(400).json({ error: "invalid_parent", message: err.message });
throw err;
}
}));
// ─── Alerts ────────────────────────────────────────────────────────────────
export const alertsBulkRouter = Router();
alertsBulkRouter.use(requireAuth, requireRole("operator"));
const previousAck = z
.object({
note: z.string().max(MAX_NOTE),
assignee: z.object({ id: z.number().int().positive().nullable(), name: z.string().max(200) }).nullable(),
ackedBy: z.string().max(200),
ackedAt: z.string().max(40),
until: z.string().max(40).nullable(),
active: z.boolean(),
})
.nullable();
const alertsBody = z.discriminatedUnion("action", [
z.object({ action: z.literal("acknowledge"), alertIds: stringIds, note: z.string().max(MAX_NOTE).default(""), assigneeId: z.number().int().positive().nullable().default(null), hours: z.number().int().min(1).max(MAX_HOURS).nullable().default(null) }),
z.object({ action: z.literal("unacknowledge"), alertIds: stringIds }),
z.object({ action: z.literal("restore_acks"), items: z.array(z.object({ alertId: z.string().min(1).max(600), previous: previousAck })).min(1).max(MAX_BULK) }),
z.object({ action: z.literal("silence"), alertIds: stringIds, minutes, reason }),
z.object({ action: z.literal("undo_silence"), created: z.array(z.number().int().positive()).max(MAX_BULK * 4), restored: z.array(z.object({ id: z.number().int().positive(), endsAt: z.string().max(40), reason: z.string().max(200).nullable() })).max(MAX_BULK * 4) }),
]);
alertsBulkRouter.post("/", asyncHandler(async (req, res) => {
const parsed = alertsBody.safeParse(req.body);
if (!parsed.success) return bad(res, parsed.error.issues[0]?.message ?? `Choose up to ${MAX_BULK} alerts and what to do.`);
const body = parsed.data;
const actor = req.currentUser!;
try {
switch (body.action) {
case "acknowledge": {
const r = await acknowledgeAlerts(body.alertIds, { note: body.note, assigneeId: body.assigneeId, hours: body.hours, actor: nameOf(req) });
await recordAudit({ actor, category: "alert", action: "bulk_ack", detail: { changed: r.changed, note: body.note, hours: body.hours, alerts: detailIds(body.alertIds) } });
return res.json(r);
}
case "unacknowledge": {
const r = await unacknowledgeAlerts(body.alertIds);
await recordAudit({ actor, category: "alert", action: "bulk_unack", detail: { changed: r.changed, alerts: detailIds(body.alertIds) } });
return res.json(r);
}
case "restore_acks": {
const changed = await restoreAcks(body.items);
await recordAudit({ actor, category: "alert", action: "bulk_undo", detail: { changed } });
return res.json({ changed, skipped: [], undo: null });
}
case "silence": {
const { targets, skipped } = await targetsOfAlerts(body.alertIds);
const r = await silenceTargets(targets, body.minutes, body.reason || "Maintenance, from several alerts", nameOf(req), false);
await recordAudit({ actor, category: "maintenance", action: "bulk_start", detail: { targets: r.silenced.map((s) => s.name).slice(0, 50), minutes: body.minutes, reason: body.reason ?? null, fromAlerts: body.alertIds.length } });
return res.json({ changed: r.silenced.length, skipped: [...skipped, ...r.skipped], undo: r.undo, silenced: r.silenced });
}
case "undo_silence": {
const changed = await undoSilence(body);
await recordAudit({ actor, category: "maintenance", action: "bulk_undo", detail: { windows: changed } });
return res.json({ changed, skipped: [], undo: null });
}
}
} catch (err) {
if (err instanceof AckError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
// ─── Hardware ──────────────────────────────────────────────────────────────
export const inventoryBulkRouter = Router();
inventoryBulkRouter.use(requireAuth, requireRole("operator"));
const dateOrBlank = z.string().max(10).nullable();
const inventoryBody = z.discriminatedUnion("action", [
z.object({ action: z.literal("change"), ids: intIds, location: z.string().max(120).nullable().optional(), warrantyUntil: dateOrBlank.optional(), endOfSupport: dateOrBlank.optional() }),
z.object({ action: z.literal("restore"), items: z.array(z.object({ id: z.number().int().positive(), location: z.string().max(120).nullable(), warrantyUntil: dateOrBlank, endOfSupport: dateOrBlank })).min(1).max(MAX_BULK) }),
]);
inventoryBulkRouter.post("/", asyncHandler(async (req, res) => {
const parsed = inventoryBody.safeParse(req.body);
if (!parsed.success) return bad(res, parsed.error.issues[0]?.message ?? `Choose up to ${MAX_BULK} pieces of hardware and what to change.`);
const body = parsed.data;
const actor = req.currentUser!;
try {
if (body.action === "restore") {
const changed = await restoreAssets(body.items);
await recordAudit({ actor, category: "inventory", action: "bulk_change", detail: { change: "undo", changed } });
return res.json({ changed, skipped: [], undo: null });
}
const { ids, action: _a, ...change } = body;
const r = await changeAssets(ids, change);
await recordAudit({ actor, category: "inventory", action: "bulk_change", detail: { ...change, changed: r.changed, ids: detailIds(ids) } });
return res.json(r);
} catch (err) {
if (err instanceof BulkInputError) return bad(res, err.message);
throw err;
}
}));
+1 -1
View File
@@ -144,7 +144,7 @@ const updateSchema = z.object({
.partial()
.optional(),
healthChecks: z
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean() })
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20) })
.partial()
.optional(),
quietHours: z
+6
View File
@@ -94,6 +94,12 @@ export async function setAck(input: SetAckInput, now: Date = new Date()): Promis
return { ack: (await loadAck(input.alertId, now))!, before };
}
/** Puts an acknowledgement back exactly as it was (the undo of a bulk change): the same note, person, times and end. */
export async function restoreAck(alertId: string, previous: AckInfo, now: Date = new Date()): Promise<void> {
const values = { note: previous.note, assigneeUserId: previous.assignee?.id ?? null, assigneeName: previous.assignee?.name ?? null, ackedBy: previous.ackedBy, ackedAt: previous.ackedAt, until: previous.until, lastSeenAt: now.toISOString() };
await db.insert(alertAcks).values({ alertKey: alertId, ...values }).onConflictDoUpdate({ target: alertAcks.alertKey, set: values });
}
export async function clearAck(alertId: string): Promise<AckInfo | null> {
const before = await loadAck(alertId);
if (!before) return null;
+145
View File
@@ -0,0 +1,145 @@
/**
* The alert noise report: which alerts fire most often, how long they last, and which have been flapping — worked out from the incident record (every time an alert
* was raised and later went away is one incident, services/incidents.ts). It is for deciding what to quieten: raise a limit, ignore a service, fix the cause.
*
* An alert is the same alert each time it comes back (the incident record keys on the alert's own id), so "fired 14 times" is the same disk or the same check
* coming and going — which is the noise. Alerts that fired only once are left out: one is an event, not noise.
*/
import { and, gte } from "drizzle-orm";
import { db } from "../db/client.js";
import { alertIncidents } from "../db/schema.js";
import { flappingKeys, FLAP_WINDOW_MS, keyOfAlertId } from "./flapGuard.js";
import { getSettings } from "./settingsStore.js";
const DAY_MS = 86_400_000;
export const NOISE_DAYS = [7, 30, 90, 180] as const;
const MAX_ROWS = 50;
/** The threshold the report uses for "flapped" when flapping is switched off in Settings (so the figure still means something). */
const DEFAULT_FLAP_RAISES = 3;
export interface NoiseRow {
/** The alert's id, which is what the incident record keys on. */
alertId: string;
/** What it said the last time. */
message: string;
category: string;
source: string;
where: string;
/** How many times it was raised in the period. */
fired: number;
totalMinutes: number;
averageMinutes: number;
longestMinutes: number;
/** The most times it was raised within any one hour. */
mostInAnHour: number;
/** Raised enough times within an hour to count as flapping. */
flapped: boolean;
/** Flapping right now: its messages are held back. */
flappingNow: boolean;
openNow: boolean;
lastAt: string;
link: string | null;
/** What could be done about it, in a sentence. */
hint: string;
}
export interface NoiseReport {
days: number;
from: string;
to: string;
/** Every incident in the period, and how many different alerts they were. */
incidents: number;
distinctAlerts: number;
/** Alerts that fired more than once, which are what the table lists. */
repeating: number;
flapped: number;
/** All the time spent with an alert up, in the period, in minutes. */
totalMinutes: number;
byCategory: { category: string; incidents: number; repeating: number }[];
rows: NoiseRow[];
}
/** Pure: the most raises within any window of `windowMs` (the times are epoch ms). */
export function mostWithin(times: number[], windowMs: number): number {
const sorted = [...times].sort((a, b) => a - b);
let best = 0;
let from = 0;
for (let i = 0; i < sorted.length; i++) {
while (sorted[i] - sorted[from] >= windowMs) from++;
best = Math.max(best, i - from + 1);
}
return best;
}
/** What to try, from the kind of alert and the key it has. Plain advice; nothing it says is done for anyone. */
export function hintFor(category: string, alertId: string): string {
const key = keyOfAlertId(alertId, category);
if (key.startsWith("new-port:")) return "If this port should be listening, mark it as expected on the server's Ports card.";
if (key.startsWith("heartbeat")) return "If the job is just sometimes late, give its heartbeat more grace; if it fails, look at what it printed.";
if (key.startsWith("endpoint:")) return "Raise “Alert after … failures in a row” on the check, or its timeout or interval — or fix what is making it fail.";
if (key.startsWith("endpoint-cert:")) return "Renew the certificate, or lower how early the check warns.";
if (key.startsWith("failed-units:")) return "If a service is expected to fail, add it to “Services to ignore” on the server's Host health card; otherwise fix the service.";
if (key.startsWith("disk-forecast:") || category === "disk") return "Raise the limit for this disk (a server can have its own, per disk), or leave it alone on the server's page if it is meant to be full.";
if (category === "offline") return "If the server is just slow to report, raise “Server offline after” under Settings → Notifications, or give this server its own limit.";
if (category === "integration") return "Raise “Alert after … consecutive failures” under Settings → Notifications; the Call log (Integrations) shows what is failing.";
if (category === "backup") return "A guest that is backed up late each time may need its “too old” limit raised on its server, or its job moved.";
return "Acknowledge it with a note, silence it with maintenance while you work on it, or fix what raises it.";
}
export async function buildNoiseReport(days: number, now: Date = new Date()): Promise<NoiseReport> {
const span = (NOISE_DAYS as readonly number[]).includes(days) ? days : 30;
const from = new Date(now.getTime() - span * DAY_MS);
const rows = await db.select().from(alertIncidents).where(and(gte(alertIncidents.openedAt, from.toISOString())));
const threshold = (await getSettings()).healthChecks.flapRaises || DEFAULT_FLAP_RAISES;
const flapping = await flappingKeys().catch(() => new Set<string>());
const groups = new Map<string, typeof rows>();
for (const r of rows) groups.set(r.alertKey, [...(groups.get(r.alertKey) ?? []), r]);
let totalMinutes = 0;
const out: NoiseRow[] = [];
const categories = new Map<string, { incidents: number; repeating: number }>();
for (const [alertId, list] of groups) {
const minutes = list.map((r) => Math.max(1, Math.round(((r.resolvedAt ? Date.parse(r.resolvedAt) : now.getTime()) - Date.parse(r.openedAt)) / 60_000)));
const sum = minutes.reduce((n, m) => n + m, 0);
totalMinutes += sum;
const latest = [...list].sort((a, b) => b.openedAt.localeCompare(a.openedAt))[0];
const c = categories.get(latest.category) ?? { incidents: 0, repeating: 0 };
c.incidents += list.length;
if (list.length > 1) c.repeating++;
categories.set(latest.category, c);
if (list.length < 2) continue;
const most = mostWithin(list.map((r) => Date.parse(r.openedAt)), FLAP_WINDOW_MS);
out.push({
alertId,
message: latest.message,
category: latest.category,
source: latest.source,
where: latest.targetName ?? latest.source,
fired: list.length,
totalMinutes: sum,
averageMinutes: Math.round(sum / list.length),
longestMinutes: Math.max(...minutes),
mostInAnHour: most,
flapped: most >= threshold,
flappingNow: flapping.has(keyOfAlertId(alertId, latest.category)),
openNow: list.some((r) => r.resolvedAt === null),
lastAt: latest.openedAt,
link: latest.link,
hint: hintFor(latest.category, alertId),
});
}
out.sort((a, b) => b.fired - a.fired || b.totalMinutes - a.totalMinutes || a.message.localeCompare(b.message));
return {
days: span,
from: from.toISOString(),
to: now.toISOString(),
incidents: rows.length,
distinctAlerts: groups.size,
repeating: out.length,
flapped: out.filter((r) => r.flapped).length,
totalMinutes,
byCategory: [...categories].map(([category, v]) => ({ category, ...v })).sort((a, b) => b.incidents - a.incidents || a.category.localeCompare(b.category)),
rows: out.slice(0, MAX_ROWS),
};
}
+2
View File
@@ -23,6 +23,8 @@ export interface Alert {
causedBy?: { name: string; alertId?: string };
/** On a host's alert: the servers on it whose own alerts are folded under this one. */
affects?: string[];
/** Raised and cleared over and over: its raised and cleared messages are being held back (services/flapGuard.ts). */
flapping?: boolean;
/** What a maintenance window would have to be on to silence it, when there is such a thing — which is what "Silence" acts on. */
target: { type: "server" | "integration" | "dns_provider"; id: number; name: string } | null;
}
+5
View File
@@ -17,6 +17,7 @@ import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresho
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
import { recordFromReport } from "./incidents.js";
import { reconcileAcks } from "./alertAcks.js";
import { flappingKeys, keyOfAlertId } from "./flapGuard.js";
import { lastAttempt as lastSelfBackup, newestSnapshot, STALE_AFTER_HOURS as SELF_BACKUP_STALE_HOURS } from "./selfBackup.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
@@ -533,6 +534,10 @@ export async function collectAlerts(): Promise<AlertsReport> {
a.message.localeCompare(b.message),
);
// Those that keep coming and going are marked, so nobody mistakes the quiet of their held messages for the problem having gone.
const flapping = await flappingKeys().catch(() => new Set<string>());
if (flapping.size > 0) for (const a of alerts) if (flapping.has(keyOfAlertId(a.id, a.category))) a.flapping = true;
const active = alerts.filter((a) => !a.silenced);
const grouped = alerts.filter((a) => a.causedBy).length;
return {
+290
View File
@@ -0,0 +1,290 @@
/**
* Bulk actions: do the same thing to several alerts, servers or hardware entries at once. Each action says what it changed and returns, with it, the body that
* puts everything back — so "Undo" is only sending that body to the same address again, and an undo needs nothing remembered on the server. Whatever could not be
* done (an alert that has gone, a server already at its limit of tags, a change that would make a loop) is listed with the reason and does not stop the rest.
*/
import { and, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, maintenanceWindows, servers } from "../db/schema.js";
import { invalidateAlertsCache, getAlerts } from "./alerts.js";
import { clearAck, restoreAck, setAck, type AckInfo, type SetAckInput } from "./alertAcks.js";
import { impactOf } from "./dependencies.js";
import { isRealDate } from "./inventory.js";
import { listActiveWindows, startOrExtendWindow } from "./maintenance.js";
import { loadParentRows, wouldLoop } from "./serverParents.js";
import { InvalidTagError, normalizeTags, parseTags } from "./serverTags.js";
export const MAX_BULK = 200;
export interface Skipped {
id: string | number;
name: string;
reason: string;
}
export interface BulkResult<U> {
changed: number;
skipped: Skipped[];
/** The body that puts it back, or null when nothing changed. */
undo: U | null;
}
const unique = <T>(list: T[]): T[] => [...new Set(list)];
// ─── Maintenance (silence), shared by servers and alerts ───────────────────
export interface SilenceUndo {
action: "undo_silence";
/** Windows these made: ended again. */
created: number[];
/** Windows that were already there and were extended: put back as they were. */
restored: { id: number; endsAt: string; reason: string | null }[];
}
type Target = { type: "server" | "integration" | "dns_provider"; id: number };
/** Puts each target under maintenance (and, if asked, what depends on a server). Targets that are already under it have their window extended. */
export async function silenceTargets(targets: Target[], minutes: number, reason: string | null, createdBy: string, withDependents: boolean, now: Date = new Date()): Promise<{ silenced: { type: string; id: number; name: string }[]; skipped: Skipped[]; undo: SilenceUndo | null }> {
const wanted = new Map<string, Target & { why: string | null }>();
for (const t of targets) wanted.set(`${t.type}:${t.id}`, { ...t, why: reason });
if (withDependents) {
for (const t of targets) {
if (t.type !== "server" && t.type !== "integration") continue;
const impact = await impactOf(t.type, t.id, now);
for (const d of impact?.dependents ?? []) {
const key = `${d.type}:${d.id}`;
if (!wanted.has(key)) wanted.set(key, { type: d.type, id: d.id, why: `Depends on ${impact?.target.name ?? "a server in the same change"}${reason ? ` — ${reason}` : ""}`.slice(0, 200) });
}
}
}
const active = await listActiveWindows(now);
const created: number[] = [];
const restored: SilenceUndo["restored"] = [];
const silenced: { type: string; id: number; name: string }[] = [];
const skipped: Skipped[] = [];
for (const t of wanted.values()) {
const existing = active.find((w) => w.targetType === t.type && w.targetId === t.id);
const started = await startOrExtendWindow(t.type, t.id, minutes, t.why, createdBy, now);
if (!started) {
skipped.push({ id: t.id, name: `${t.type} #${t.id}`, reason: "it no longer exists" });
continue;
}
silenced.push({ type: t.type, id: t.id, name: started.target.name });
if (started.extended && existing) restored.push({ id: existing.id, endsAt: existing.endsAt, reason: existing.reason });
else created.push(started.window.id);
}
invalidateAlertsCache();
return { silenced, skipped, undo: silenced.length > 0 ? { action: "undo_silence", created, restored } : null };
}
export async function undoSilence(undo: Pick<SilenceUndo, "created" | "restored">, now: Date = new Date()): Promise<number> {
if (undo.created.length > 0) await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(inArray(maintenanceWindows.id, undo.created));
for (const r of undo.restored) await db.update(maintenanceWindows).set({ endsAt: r.endsAt, reason: r.reason }).where(eq(maintenanceWindows.id, r.id));
invalidateAlertsCache();
return undo.created.length + undo.restored.length;
}
// ─── Servers ───────────────────────────────────────────────────────────────
export interface TagsUndo {
action: "set_tags";
items: { id: number; tags: string[] }[];
}
/** Adds or takes away a tag on each server. A server that already has it (or lacks it), or has no room for another, is left as it was. */
export async function changeServerTag(ids: number[], change: { add: string } | { remove: string }): Promise<BulkResult<TagsUndo>> {
const rows = await db.select().from(servers).where(inArray(servers.id, unique(ids)));
const tag = normalizeTags(["add" in change ? change.add : change.remove])[0];
if (!tag) throw new InvalidTagError("A tag needs a name.");
const skipped: Skipped[] = unique(ids).filter((id) => !rows.some((r) => r.id === id)).map((id) => ({ id, name: `server #${id}`, reason: "it no longer exists" }));
const items: TagsUndo["items"] = [];
for (const r of rows) {
const before = parseTags(r.tags);
let after: string[];
if ("add" in change) {
if (before.includes(tag)) {
skipped.push({ id: r.id, name: r.name, reason: "it already has that tag" });
continue;
}
try {
after = normalizeTags([...before, tag]);
} catch (err) {
if (err instanceof InvalidTagError) {
skipped.push({ id: r.id, name: r.name, reason: err.message });
continue;
}
throw err;
}
} else {
if (!before.includes(tag)) {
skipped.push({ id: r.id, name: r.name, reason: "it doesn't have that tag" });
continue;
}
after = before.filter((t) => t !== tag);
}
await db.update(servers).set({ tags: after.length > 0 ? JSON.stringify(after) : null }).where(eq(servers.id, r.id));
items.push({ id: r.id, tags: before });
}
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "set_tags", items } : null };
}
/** Sets each server's tags to exactly what is given (the undo of a tag change). */
export async function setServerTags(items: { id: number; tags: string[] }[]): Promise<number> {
let n = 0;
for (const item of items) {
const tags = normalizeTags(item.tags);
const done = await db.update(servers).set({ tags: tags.length > 0 ? JSON.stringify(tags) : null }).where(eq(servers.id, item.id)).returning({ id: servers.id });
n += done.length;
}
return n;
}
export interface ParentUndo {
action: "restore_parents";
items: { id: number; parentServerId: number | null; noParent: boolean }[];
}
/** Sets which server each runs on. `parentServerId` null goes back to working it out from the Proxmox node; a change that would make a loop is skipped. */
export async function setServerParents(ids: number[], parentServerId: number | null): Promise<BulkResult<ParentUndo>> {
const rows = await loadParentRows();
const skipped: Skipped[] = [];
const items: ParentUndo["items"] = [];
if (parentServerId !== null && !rows.some((r) => r.id === parentServerId)) throw new RangeError("That server doesn't exist.");
for (const id of unique(ids)) {
const row = rows.find((r) => r.id === id);
if (!row) {
skipped.push({ id, name: `server #${id}`, reason: "it no longer exists" });
continue;
}
if (parentServerId !== null) {
if (parentServerId === id) {
skipped.push({ id, name: row.name, reason: "a server can't run on itself" });
continue;
}
if (wouldLoop(rows, id, parentServerId)) {
skipped.push({ id, name: row.name, reason: "that server already runs on this one, so each would be waiting for the other" });
continue;
}
}
items.push({ id, parentServerId: row.parentServerId ?? null, noParent: row.noParent ?? false });
await db.update(servers).set({ parentServerId, noParent: false }).where(eq(servers.id, id));
row.parentServerId = parentServerId;
row.noParent = false;
}
if (items.length > 0) invalidateAlertsCache();
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_parents", items } : null };
}
export async function restoreServerParents(items: ParentUndo["items"]): Promise<number> {
let n = 0;
for (const i of items) {
const done = await db.update(servers).set({ parentServerId: i.parentServerId, noParent: i.noParent }).where(eq(servers.id, i.id)).returning({ id: servers.id });
n += done.length;
}
if (n > 0) invalidateAlertsCache();
return n;
}
// ─── Alerts ────────────────────────────────────────────────────────────────
export interface AcksUndo {
action: "restore_acks";
items: { alertId: string; previous: AckInfo | null }[];
}
/** Acknowledges each alert the same way. Alerts that are no longer there are skipped. */
export async function acknowledgeAlerts(alertIds: string[], input: Omit<SetAckInput, "alertId">): Promise<BulkResult<AcksUndo>> {
const present = new Map((await getAlerts(false)).alerts.map((a) => [a.id, a]));
const skipped: Skipped[] = [];
const items: AcksUndo["items"] = [];
for (const alertId of unique(alertIds)) {
if (!present.has(alertId)) {
skipped.push({ id: alertId, name: alertId, reason: "it isn't there any more" });
continue;
}
const { before } = await setAck({ ...input, alertId });
items.push({ alertId, previous: before });
}
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_acks", items } : null };
}
/** Takes the acknowledgement off each alert that has one. */
export async function unacknowledgeAlerts(alertIds: string[]): Promise<BulkResult<AcksUndo>> {
const skipped: Skipped[] = [];
const items: AcksUndo["items"] = [];
for (const alertId of unique(alertIds)) {
const before = await clearAck(alertId);
if (!before) {
skipped.push({ id: alertId, name: alertId, reason: "nobody had acknowledged it" });
continue;
}
items.push({ alertId, previous: before });
}
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_acks", items } : null };
}
/** Puts each alert's acknowledgement back as it was (or takes it off, if it had none). */
export async function restoreAcks(items: AcksUndo["items"]): Promise<number> {
for (const i of items) {
if (i.previous === null) await clearAck(i.alertId);
else await restoreAck(i.alertId, i.previous);
}
return items.length;
}
/** The maintenance targets of the alerts given (those with none can't be silenced). */
export async function targetsOfAlerts(alertIds: string[]): Promise<{ targets: Target[]; skipped: Skipped[] }> {
const present = new Map((await getAlerts(false)).alerts.map((a) => [a.id, a]));
const targets = new Map<string, Target>();
const skipped: Skipped[] = [];
for (const alertId of unique(alertIds)) {
const alert = present.get(alertId);
if (!alert) skipped.push({ id: alertId, name: alertId, reason: "it isn't there any more" });
else if (!alert.target) skipped.push({ id: alertId, name: alert.message.slice(0, 80), reason: "there is nothing to put under maintenance for it" });
else targets.set(`${alert.target.type}:${alert.target.id}`, { type: alert.target.type, id: alert.target.id });
}
return { targets: [...targets.values()], skipped };
}
// ─── Hardware ──────────────────────────────────────────────────────────────
export interface AssetsUndo {
action: "restore";
items: { id: number; location: string | null; warrantyUntil: string | null; endOfSupport: string | null }[];
}
export class BulkInputError extends Error {}
/** What a blank in a form means: nothing, not an empty string. */
const blank = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
/** Sets the location and/or the dates of each piece of hardware. Whatever is not given is left as it is; a blank clears. */
export async function changeAssets(ids: number[], change: { location?: string | null; warrantyUntil?: string | null; endOfSupport?: string | null }): Promise<BulkResult<AssetsUndo>> {
const set: Partial<typeof assets.$inferInsert> = {};
if (change.location !== undefined) set.location = blank(change.location);
for (const field of ["warrantyUntil", "endOfSupport"] as const) {
if (change[field] === undefined) continue;
const v = blank(change[field]);
if (v !== null && !isRealDate(v)) throw new BulkInputError("Use a real date, like 2027-06-30.");
set[field] = v;
}
if (Object.keys(set).length === 0) throw new BulkInputError("Say what to change.");
const rows = await db.select().from(assets).where(inArray(assets.id, unique(ids)));
const skipped: Skipped[] = unique(ids).filter((id) => !rows.some((r) => r.id === id)).map((id) => ({ id, name: `hardware #${id}`, reason: "it no longer exists" }));
const items: AssetsUndo["items"] = [];
for (const r of rows) {
items.push({ id: r.id, location: r.location, warrantyUntil: r.warrantyUntil, endOfSupport: r.endOfSupport });
await db.update(assets).set({ ...set, updatedAt: new Date().toISOString() }).where(eq(assets.id, r.id));
}
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore", items } : null };
}
export async function restoreAssets(items: AssetsUndo["items"]): Promise<number> {
let n = 0;
for (const i of items) {
const done = await db.update(assets).set({ location: i.location, warrantyUntil: i.warrantyUntil, endOfSupport: i.endOfSupport, updatedAt: new Date().toISOString() }).where(and(eq(assets.id, i.id))).returning({ id: assets.id });
n += done.length;
}
return n;
}
+22
View File
@@ -157,6 +157,28 @@ export function describeChange(category: string, action: string, detail: Record<
return { title: `Endpoint check changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
case "endpoint/delete":
return { title: `Endpoint check removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "server/bulk_tag": {
const n = typeof detail.changed === "number" ? detail.changed : null;
return { title: detail.change === "undo" ? "A tag change on several servers was undone" : `${str(detail.change) === "remove_tag" ? "Tag taken off" : "Tag added to"} ${n ?? "several"} server${n === 1 ? "" : "s"}${str(detail.tag) ? `: ${str(detail.tag)}` : ""}`, tone: "neutral", detail: null };
}
case "server/bulk_runs_on":
return { title: detail.change === "undo" ? "A change of which server others run on was undone" : `Which server ${typeof detail.changed === "number" ? detail.changed : "several"} run on was changed`, tone: "neutral", detail: null };
case "maintenance/bulk_start":
return { title: `Maintenance started on several at once${Array.isArray(detail.servers ?? detail.targets) ? `: ${((detail.servers ?? detail.targets) as unknown[]).slice(0, 4).join(", ")}` : ""}`, tone: "neutral", detail: typeof detail.reason === "string" ? detail.reason : null };
case "maintenance/bulk_undo":
return { title: "Maintenance started on several at once was undone", tone: "neutral", detail: null };
case "alert/bulk_ack":
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} alert${detail.changed === 1 ? "" : "s"} acknowledged at once`, tone: "neutral", detail: typeof detail.note === "string" && detail.note ? detail.note : null };
case "alert/bulk_unack":
return { title: `Acknowledgement taken off ${typeof detail.changed === "number" ? detail.changed : "several"} alert${detail.changed === 1 ? "" : "s"}`, tone: "neutral", detail: null };
case "alert/bulk_undo":
return { title: "Acknowledging several alerts at once was undone", tone: "neutral", detail: null };
case "inventory/bulk_change":
return { title: detail.change === "undo" ? "A change to several pieces of hardware was undone" : `${typeof detail.changed === "number" ? detail.changed : "Several"} piece${detail.changed === 1 ? "" : "s"} of hardware changed at once`, tone: "neutral", detail: genericSummary(detail, ["ids", "changed", "change"]) };
case "alert/flapping_started":
return { title: `Alert began flapping${str(detail.message) ? `: ${str(detail.message)}` : ""}`, tone: "bad", detail: typeof detail.raises === "number" ? `raised ${detail.raises} times within an hour; its messages are held back` : null };
case "alert/flapping_ended":
return { title: `Alert settled${str(detail.message) ? `: ${str(detail.message)}` : ""}`, tone: detail.active === true ? "bad" : "good", detail: detail.active === true ? "steady for an hour, and still active" : "steady for an hour, and cleared" };
case "report/send_monthly":
return { title: `Monthly report sent${str(detail.month) ? ` (${str(detail.month)})` : ""}`, tone: "neutral", detail: null };
case "patch_plan/create":
+161
View File
@@ -0,0 +1,161 @@
/**
* Flapping: an alert that is raised and clears over and over — a service that crashes and restarts, a link that drops every few minutes — would send a raised and a
* recovered message each time, which is noise that hides the one thing worth knowing: that it is unstable. So when the same alert has been raised several times within
* an hour, it is called *flapping*: one message says so, the Alerts page marks it, and its further raised and recovered messages are held back. When it has been quiet
* for an hour it has settled, and one message says what state it is in.
*
* It sits in the notification layer (notifyHealthIssues / notifyHealthRecovered), so it covers everything that goes through the health monitor's bookkeeping: servers,
* disks, services, new ports, endpoint checks and heartbeats. The alert itself is not touched — it is still raised, counted, and in the incident record — only what is
* *sent* is calmer. The event webhooks, which are for machines, still hear every raise and clear.
*/
import { recordAudit } from "./audit.js";
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
const FLAG = "flapState";
/** The raises that count are those within this long. */
export const FLAP_WINDOW_MS = 60 * 60_000;
/** A flapping alert that has had no raise or clear for this long has settled. */
export const SETTLE_MS = 60 * 60_000;
export interface FlapEntry {
/** When it was raised (and announced), epoch ms, within the window. */
raises: number[];
flapping: boolean;
/** When it was called flapping. */
since: number | null;
/** The last time it was raised or cleared while flapping. */
lastEvent: number;
/** What it says, for the messages that come later. */
message: string;
}
export type FlapState = Record<string, FlapEntry>;
export type RaiseVerdict = "pass" | "started" | "held";
/** Pure: what a raise does to an alert's record — and whether its message goes out, becomes the one "flapping" message, or is held. A threshold of 0 turns it off. */
export function registerRaise(entry: FlapEntry | undefined, now: number, threshold: number, message: string): { entry: FlapEntry; verdict: RaiseVerdict } {
const kept = (entry?.raises ?? []).filter((t) => now - t < FLAP_WINDOW_MS);
if (entry?.flapping) return { entry: { ...entry, raises: [...kept, now], lastEvent: now, message }, verdict: "held" };
const raises = [...kept, now];
if (threshold > 0 && raises.length >= threshold) return { entry: { raises, flapping: true, since: now, lastEvent: now, message }, verdict: "started" };
return { entry: { raises, flapping: false, since: null, lastEvent: now, message }, verdict: "pass" };
}
/** Pure: whether a recovery message is held (the alert is flapping), and the record after it. */
export function registerRecovery(entry: FlapEntry | undefined, now: number): { entry: FlapEntry | undefined; hold: boolean } {
if (!entry?.flapping) return { entry, hold: false };
return { entry: { ...entry, lastEvent: now }, hold: true };
}
/** Pure: whether a flapping alert has been quiet long enough to have settled. */
export const hasSettled = (entry: FlapEntry, now: number) => entry.flapping && now - entry.lastEvent >= SETTLE_MS;
// ─── What is kept ──────────────────────────────────────────────────────────
async function load(): Promise<FlapState> {
const raw = await getInternalFlag(FLAG);
if (!raw) return {};
try {
const v = JSON.parse(raw);
return v && typeof v === "object" ? (v as FlapState) : {};
} catch {
return {};
}
}
async function save(state: FlapState, now: number): Promise<void> {
// An alert that is not flapping and has had no raise within the window has nothing worth remembering.
const next: FlapState = {};
for (const [key, e] of Object.entries(state)) if (e.flapping || e.raises.some((t) => now - t < FLAP_WINDOW_MS)) next[key] = e;
await setInternalFlag(FLAG, JSON.stringify(next));
}
// Two monitors can announce at the same moment; each reads the record, changes it and writes it back, so they take turns.
let turn: Promise<unknown> = Promise.resolve();
function inTurn<T>(work: () => Promise<T>): Promise<T> {
const result = turn.then(work, work);
turn = result.catch(() => undefined);
return result;
}
const thresholdNow = async () => (await getSettings()).healthChecks.flapRaises;
export interface Keyed {
key?: string;
message: string;
}
/**
* Looks at alerts that have just been raised: those that go out as they are, and those that have just begun flapping (each of which goes out once, as the
* "flapping" message, with how many times it has been raised). Held ones are in neither list.
*/
export async function dampRaised<T extends Keyed>(issues: T[], now: Date = new Date()): Promise<{ pass: T[]; started: (T & { count: number })[] }> {
const withKey = issues.filter((i) => i.key);
if (withKey.length === 0) return { pass: issues, started: [] };
return inTurn(async () => {
const threshold = await thresholdNow();
const state = await load();
const pass: T[] = [];
const started: (T & { count: number })[] = [];
for (const issue of issues) {
if (!issue.key) {
pass.push(issue);
continue;
}
const { entry, verdict } = registerRaise(state[issue.key], now.getTime(), threshold, issue.message);
state[issue.key] = entry;
if (verdict === "pass") pass.push(issue);
else if (verdict === "started") {
started.push({ ...issue, count: entry.raises.length });
await recordAudit({ category: "alert", action: "flapping_started", detail: { key: issue.key, message: issue.message, raises: entry.raises.length } }).catch(() => undefined);
}
}
await save(state, now.getTime());
return { pass, started };
});
}
/** Looks at alerts that have just cleared: those of flapping alerts are held (their moment is noted, so the quiet is counted from it). */
export async function dampRecovered<T extends { key?: string }>(resolved: T[], now: Date = new Date()): Promise<T[]> {
if (!resolved.some((r) => r.key)) return resolved;
return inTurn(async () => {
const state = await load();
const pass: T[] = [];
for (const r of resolved) {
if (!r.key) {
pass.push(r);
continue;
}
const { entry, hold } = registerRecovery(state[r.key], now.getTime());
if (entry) state[r.key] = entry;
if (!hold) pass.push(r);
}
await save(state, now.getTime());
return pass;
});
}
/** The alerts that have been quiet for an hour and so have settled — dropped from the record, with whether each is active now. */
export async function settleFlaps(activeKeys: Set<string>, now: Date = new Date()): Promise<{ key: string; message: string; active: boolean; raises: number }[]> {
return inTurn(async () => {
const state = await load();
const settled: { key: string; message: string; active: boolean; raises: number }[] = [];
for (const [key, e] of Object.entries(state)) {
if (!hasSettled(e, now.getTime())) continue;
settled.push({ key, message: e.message, active: activeKeys.has(key), raises: e.raises.length });
delete state[key];
await recordAudit({ category: "alert", action: "flapping_ended", detail: { key, message: e.message, active: activeKeys.has(key) } }).catch(() => undefined);
}
if (settled.length > 0) await save(state, now.getTime());
return settled;
});
}
/** The alerts that are flapping now (by the key the health checks give them). */
export async function flappingKeys(): Promise<Set<string>> {
const state = await load();
return new Set(Object.entries(state).filter(([, e]) => e.flapping).map(([key]) => key));
}
/** The key a health condition has behind an alert's id ("disk:" + "disk:server:3:/" is the id of the alert about "disk:server:3:/"). */
export const keyOfAlertId = (id: string, category: string) => (id.startsWith(`${category}:`) ? id.slice(category.length + 1) : id);
+23
View File
@@ -0,0 +1,23 @@
/**
* Every few minutes: flapping alerts that have been quiet for an hour have settled — say so once, with the state each is in now (see services/flapGuard.ts).
*/
import { getAlerts } from "./alerts.js";
import { keyOfAlertId, settleFlaps } from "./flapGuard.js";
import { notifySettled } from "./notify.js";
/** One look: returns how many settled. */
export async function settleFlappingAlerts(now: Date = new Date()): Promise<number> {
const report = await getAlerts(false);
const active = new Set(report.alerts.map((a) => keyOfAlertId(a.id, a.category)));
const settled = await settleFlaps(active, now);
if (settled.length > 0) await notifySettled(settled);
return settled.length;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initFlapMonitor(): void {
if (timer) return;
timer = setInterval(() => void settleFlappingAlerts().catch((err) => console.error("[flap] couldn't look at the flapping alerts:", err)), 5 * 60_000);
timer.unref();
}
+27
View File
@@ -7,6 +7,7 @@ import { db } from "../db/client.js";
import { inArray } from "drizzle-orm";
import { pushSubscriptions, servers } from "../db/schema.js";
import { parseTags } from "./serverTags.js";
import { dampRaised, dampRecovered } from "./flapGuard.js";
import { firstMatchingRule, metaOfConditionKey, pushAccepts, type ChannelId, type NotifyMeta } from "./notifyRouting.js";
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
@@ -483,7 +484,12 @@ export async function notifyHealthIssues(issues: { message: string; key?: string
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
// An alert that keeps coming and going is announced once as "flapping" and then left quiet (services/flapGuard.ts); that is tracked whether or not this is switched on.
const { pass, started } = await dampRaised(issues);
if (!(await eventEnabled("healthAlerts"))) return;
if (started.length > 0) await notifyFlapping(started);
if (pass.length === 0) return;
issues = pass;
await notifyGrouped(issues, issueMeta, (group) => ({
title: "Homelab Manager — Health Alert",
message: `${group.length} new issue${group.length !== 1 ? "s" : ""}:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`,
@@ -493,13 +499,34 @@ export async function notifyHealthIssues(issues: { message: string; key?: string
export async function notifyHealthRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
const passed = await dampRecovered(resolved);
if (!(await eventEnabled("healthAlerts"))) return;
if (passed.length === 0) return;
resolved = passed;
await notifyGrouped(resolved, issueMeta, (group) => ({
title: "Homelab Manager — Health Recovered",
message: `${group.length} issue${group.length !== 1 ? "s" : ""} cleared:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
}));
}
/** An alert that has just been called flapping: said once, with how often it has come, instead of each time. */
export async function notifyFlapping(items: { message: string; key?: string; severity?: string; count: number }[]): Promise<void> {
if (items.length === 0) return;
await notifyGrouped(items, issueMeta, (group) => ({
title: "Homelab Manager — Flapping alert",
message: `${group.length === 1 ? "An alert keeps" : `${group.length} alerts keep`} coming and going:\n\n${group.map((i) => `↻ ${i.message} — raised ${i.count} times within an hour`).join("\n")}\n\nIts further messages are held back until it has been quiet for an hour; it stays on the Alerts page, marked as flapping.`,
}));
}
/** Flapping alerts that have been quiet for an hour: what state each is in now. */
export async function notifySettled(items: { message: string; key: string; active: boolean }[]): Promise<void> {
if (items.length === 0 || !(await eventEnabled("healthAlerts"))) return;
await notifyGrouped(items, issueMeta, (group) => ({
title: "Homelab Manager — Alert settled",
message: `${group.length === 1 ? "An alert that was flapping has" : `${group.length} alerts that were flapping have`} been steady for an hour:\n\n${group.map((i) => `${i.active ? "⚠ still active" : "✓ cleared"} — ${i.message}`).join("\n")}`,
}));
}
export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
+3 -1
View File
@@ -115,6 +115,8 @@ export interface HealthCheckSettings {
restoreTestDays: number;
/** A server that starts listening on a TCP port it did not at first, and that is not noted on its Ports card, is a warning (services/newPorts.ts). */
newPortAlerts: boolean;
/** An alert raised this many times within an hour is "flapping": one message says so and its further raised/cleared messages are held until it has been quiet for an hour. 0 turns it off. */
flapRaises: number;
}
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
@@ -285,7 +287,7 @@ const DEFAULTS: AppSettings = {
selfBackup: { enabled: true, time: "03:30", keep: 14 },
profilePictures: { gravatar: false, identityProvider: false },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3 },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
consistency: { excludedRanges: ["172.16.0.0/12"] },