Flapping alerts, a noise report and bulk actions with undo

Flapping: an alert raised three times within an hour is announced once as
flapping, its further raise/clear messages are held until it has been calm
for an hour, and one message then says whether it is still active. Event
webhooks still hear everything. The state survives restarts.

Alerts > Noise lists the alerts that keep coming back (7-180 days): how often,
how long, most in an hour, and a hint on what helps.

Bulk actions on Alerts (acknowledge, remove acknowledgement, silence), Servers
(tag, silence, runs on) and Inventory (location and dates, print labels). Each
reports what it left alone and why, and has an Undo on the toast.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 23:55:09 +02:00
1 parent be6c5552bd
commit 73a06ebe63
35 files changed
+2546 -17

No files matched your search

+5
View File
@@ -52,6 +52,7 @@ import { heartbeatsRouter } from "./routes/heartbeats.js";
import { agendaRouter } from "./routes/agenda.js";
import { patchPlansRouter } from "./routes/patchPlans.js";
import { reportsRouter } from "./routes/reports.js";
import { alertsBulkRouter, inventoryBulkRouter, serversBulkRouter } from "./routes/bulk.js";
import { publicCalendarRouter } from "./routes/publicCalendar.js";
import { heartbeatPingRouter } from "./routes/heartbeatPing.js";
import { apiErrorHandler } from "./utils/apiErrorHandler.js";
@@ -123,6 +124,8 @@ export function createApp() {
app.use("/api/update-runs", updateRunsRouter);
app.use("/api/db-backups", dbBackupsRouter);
app.use("/api/incidents", incidentsRouter);
// The bulk routes come before the routers of the same things, so "/bulk" is not read as an id.
app.use("/api/inventory/bulk", inventoryBulkRouter);
app.use("/api/inventory", inventoryRouter);
app.use("/api/dependencies", dependenciesRouter);
app.use("/api/push", pushRouter);
@@ -142,6 +145,7 @@ export function createApp() {
app.use("/api/secrets", secretsRouter);
app.use("/api/ipam", ipamRouter);
app.use("/api/dns", dnsRouter);
app.use("/api/servers/bulk", serversBulkRouter);
app.use("/api/servers", serversRouter);
app.use("/api/tasks", tasksRouter);
app.use("/api/integrations", integrationsRouter);
@@ -154,6 +158,7 @@ export function createApp() {
app.use("/api/privacy", privacyRouter);
app.use("/api/tags", tagsRouter);
app.use("/api/ports", portsRouter);
app.use("/api/alerts/bulk", alertsBulkRouter);
app.use("/api/alerts", alertsRouter);
app.use("/api/map", mapRouter);
app.use("/api/generator", generatorRouter);
+2
View File
@@ -23,6 +23,7 @@ import { initUpdateRuns } from "./services/updateRuns.js";
import { initEscalation } from "./services/escalation.js";
import { initEndpointMonitor } from "./services/endpointMonitor.js";
import { initPatchPlans } from "./services/patchPlans.js";
import { initFlapMonitor } from "./services/flapMonitor.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
import { initIncidentRecorder } from "./services/incidents.js";
import { getAlerts } from "./services/alerts.js";
@@ -65,6 +66,7 @@ initIncidentRecorder(() => getAlerts(false));
initEscalation();
initEndpointMonitor();
initPatchPlans();
initFlapMonitor();
createApp().listen(env.port, () => {
console.log(`homelab-manager listening on port ${env.port}`);
+7
View File
@@ -5,6 +5,7 @@ import { getAlerts } from "../services/alerts.js";
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
import { recordAudit } from "../services/audit.js";
import { openIncidentIds } from "../services/incidents.js";
import { buildNoiseReport, NOISE_DAYS } from "../services/alertNoise.js";
import { ticketsForAlerts, type AlertTicketInfo } from "../services/alertTickets.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -41,6 +42,12 @@ alertsRouter.get("/", asyncHandler(async (req, res) => {
});
}));
// Which alerts fire most often and how long they last, from the incident record — for everyone signed in, like the history it comes from. ?days=7|30|90|180.
alertsRouter.get("/noise", asyncHandler(async (req, res) => {
const asked = Number(req.query.days);
res.json(await buildNoiseReport((NOISE_DAYS as readonly number[]).includes(asked) ? asked : 30));
}));
// Who an alert can be handed to.
alertsRouter.get("/assignees", requireRole("operator"), asyncHandler(async (_req, res) => {
res.json({ users: await assignableUsers() });
+184
View File
@@ -0,0 +1,184 @@
import { Router } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { AckError, MAX_HOURS, MAX_NOTE } from "../services/alertAcks.js";
import { recordAudit } from "../services/audit.js";
import { acknowledgeAlerts, BulkInputError, changeAssets, changeServerTag, MAX_BULK, restoreAcks, restoreAssets, restoreServerParents, setServerParents, setServerTags, silenceTargets, targetsOfAlerts, undoSilence, unacknowledgeAlerts } from "../services/bulkActions.js";
import { InvalidTagError } from "../services/serverTags.js";
import { asyncHandler } from "../utils/asyncHandler.js";
// Bulk actions are the single-item ones done to several at once, so each needs the same role the single one does; and every one returns the body that puts it back,
// which is all an "Undo" sends. One audit entry says what was done to how many.
const intIds = z.array(z.number().int().positive()).min(1).max(MAX_BULK);
const stringIds = z.array(z.string().min(1).max(600)).min(1).max(MAX_BULK);
const minutes = z.number().int().min(5).max(7 * 24 * 60);
const reason = z.string().trim().max(200).optional();
const detailIds = (ids: unknown[]) => ids.slice(0, 50);
const bad = (res: import("express").Response, message: string) => res.status(400).json({ error: "invalid_body", message });
const nameOf = (req: import("express").Request) => {
const u = req.currentUser!;
return u.name ?? u.email ?? u.oidcSub;
};
// ─── Servers ───────────────────────────────────────────────────────────────
export const serversBulkRouter = Router();
serversBulkRouter.use(requireAuth);
const serversBody = z.discriminatedUnion("action", [
z.object({ action: z.literal("add_tag"), ids: intIds, tag: z.string().min(1).max(100) }),
z.object({ action: z.literal("remove_tag"), ids: intIds, tag: z.string().min(1).max(100) }),
z.object({ action: z.literal("set_tags"), items: z.array(z.object({ id: z.number().int().positive(), tags: z.array(z.string().max(100)).max(50) })).min(1).max(MAX_BULK) }),
z.object({ action: z.literal("silence"), ids: intIds, minutes, reason, withDependents: z.boolean().optional() }),
z.object({ action: z.literal("undo_silence"), created: z.array(z.number().int().positive()).max(MAX_BULK * 4), restored: z.array(z.object({ id: z.number().int().positive(), endsAt: z.string().max(40), reason: z.string().max(200).nullable() })).max(MAX_BULK * 4) }),
z.object({ action: z.literal("set_parent"), ids: intIds, parentServerId: z.number().int().positive().nullable() }),
z.object({ action: z.literal("restore_parents"), items: z.array(z.object({ id: z.number().int().positive(), parentServerId: z.number().int().positive().nullable(), noParent: z.boolean() })).min(1).max(MAX_BULK) }),
]);
serversBulkRouter.post("/", asyncHandler(async (req, res) => {
const parsed = serversBody.safeParse(req.body);
if (!parsed.success) return bad(res, parsed.error.issues[0]?.message ?? `Choose up to ${MAX_BULK} servers and what to do.`);
const body = parsed.data;
const actor = req.currentUser!;
// Tags and maintenance are an operator's, like one server's; which server another runs on is an administrator's.
const needsAdmin = body.action === "set_parent" || body.action === "restore_parents";
if (actor.role !== "admin" && (needsAdmin || actor.role !== "operator")) return res.status(403).json({ error: "forbidden", message: needsAdmin ? "Changing which server another runs on needs an administrator." : "That needs an operator." });
try {
switch (body.action) {
case "add_tag":
case "remove_tag": {
const r = await changeServerTag(body.ids, body.action === "add_tag" ? { add: body.tag } : { remove: body.tag });
await recordAudit({ actor, category: "server", action: "bulk_tag", detail: { change: body.action, tag: body.tag, changed: r.changed, ids: detailIds(body.ids) } });
return res.json(r);
}
case "set_tags": {
const changed = await setServerTags(body.items);
await recordAudit({ actor, category: "server", action: "bulk_tag", detail: { change: "undo", changed, ids: detailIds(body.items.map((i) => i.id)) } });
return res.json({ changed, skipped: [], undo: null });
}
case "silence": {
const r = await silenceTargets(body.ids.map((id) => ({ type: "server" as const, id })), body.minutes, body.reason || "Maintenance on several servers", nameOf(req), body.withDependents === true);
await recordAudit({ actor, category: "maintenance", action: "bulk_start", detail: { servers: r.silenced.map((s) => s.name).slice(0, 50), minutes: body.minutes, reason: body.reason ?? null, withDependents: body.withDependents === true } });
return res.json({ changed: r.silenced.length, skipped: r.skipped, undo: r.undo, silenced: r.silenced });
}
case "undo_silence": {
const changed = await undoSilence(body);
await recordAudit({ actor, category: "maintenance", action: "bulk_undo", detail: { windows: changed } });
return res.json({ changed, skipped: [], undo: null });
}
case "set_parent": {
const r = await setServerParents(body.ids, body.parentServerId);
await recordAudit({ actor, category: "server", action: "bulk_runs_on", detail: { parentServerId: body.parentServerId, changed: r.changed, ids: detailIds(body.ids) } });
return res.json(r);
}
case "restore_parents": {
const changed = await restoreServerParents(body.items);
await recordAudit({ actor, category: "server", action: "bulk_runs_on", detail: { change: "undo", changed } });
return res.json({ changed, skipped: [], undo: null });
}
}
} catch (err) {
if (err instanceof InvalidTagError) return res.status(400).json({ error: "invalid_tag", message: err.message });
if (err instanceof RangeError) return res.status(400).json({ error: "invalid_parent", message: err.message });
throw err;
}
}));
// ─── Alerts ────────────────────────────────────────────────────────────────
export const alertsBulkRouter = Router();
alertsBulkRouter.use(requireAuth, requireRole("operator"));
const previousAck = z
.object({
note: z.string().max(MAX_NOTE),
assignee: z.object({ id: z.number().int().positive().nullable(), name: z.string().max(200) }).nullable(),
ackedBy: z.string().max(200),
ackedAt: z.string().max(40),
until: z.string().max(40).nullable(),
active: z.boolean(),
})
.nullable();
const alertsBody = z.discriminatedUnion("action", [
z.object({ action: z.literal("acknowledge"), alertIds: stringIds, note: z.string().max(MAX_NOTE).default(""), assigneeId: z.number().int().positive().nullable().default(null), hours: z.number().int().min(1).max(MAX_HOURS).nullable().default(null) }),
z.object({ action: z.literal("unacknowledge"), alertIds: stringIds }),
z.object({ action: z.literal("restore_acks"), items: z.array(z.object({ alertId: z.string().min(1).max(600), previous: previousAck })).min(1).max(MAX_BULK) }),
z.object({ action: z.literal("silence"), alertIds: stringIds, minutes, reason }),
z.object({ action: z.literal("undo_silence"), created: z.array(z.number().int().positive()).max(MAX_BULK * 4), restored: z.array(z.object({ id: z.number().int().positive(), endsAt: z.string().max(40), reason: z.string().max(200).nullable() })).max(MAX_BULK * 4) }),
]);
alertsBulkRouter.post("/", asyncHandler(async (req, res) => {
const parsed = alertsBody.safeParse(req.body);
if (!parsed.success) return bad(res, parsed.error.issues[0]?.message ?? `Choose up to ${MAX_BULK} alerts and what to do.`);
const body = parsed.data;
const actor = req.currentUser!;
try {
switch (body.action) {
case "acknowledge": {
const r = await acknowledgeAlerts(body.alertIds, { note: body.note, assigneeId: body.assigneeId, hours: body.hours, actor: nameOf(req) });
await recordAudit({ actor, category: "alert", action: "bulk_ack", detail: { changed: r.changed, note: body.note, hours: body.hours, alerts: detailIds(body.alertIds) } });
return res.json(r);
}
case "unacknowledge": {
const r = await unacknowledgeAlerts(body.alertIds);
await recordAudit({ actor, category: "alert", action: "bulk_unack", detail: { changed: r.changed, alerts: detailIds(body.alertIds) } });
return res.json(r);
}
case "restore_acks": {
const changed = await restoreAcks(body.items);
await recordAudit({ actor, category: "alert", action: "bulk_undo", detail: { changed } });
return res.json({ changed, skipped: [], undo: null });
}
case "silence": {
const { targets, skipped } = await targetsOfAlerts(body.alertIds);
const r = await silenceTargets(targets, body.minutes, body.reason || "Maintenance, from several alerts", nameOf(req), false);
await recordAudit({ actor, category: "maintenance", action: "bulk_start", detail: { targets: r.silenced.map((s) => s.name).slice(0, 50), minutes: body.minutes, reason: body.reason ?? null, fromAlerts: body.alertIds.length } });
return res.json({ changed: r.silenced.length, skipped: [...skipped, ...r.skipped], undo: r.undo, silenced: r.silenced });
}
case "undo_silence": {
const changed = await undoSilence(body);
await recordAudit({ actor, category: "maintenance", action: "bulk_undo", detail: { windows: changed } });
return res.json({ changed, skipped: [], undo: null });
}
}
} catch (err) {
if (err instanceof AckError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
// ─── Hardware ──────────────────────────────────────────────────────────────
export const inventoryBulkRouter = Router();
inventoryBulkRouter.use(requireAuth, requireRole("operator"));
const dateOrBlank = z.string().max(10).nullable();
const inventoryBody = z.discriminatedUnion("action", [
z.object({ action: z.literal("change"), ids: intIds, location: z.string().max(120).nullable().optional(), warrantyUntil: dateOrBlank.optional(), endOfSupport: dateOrBlank.optional() }),
z.object({ action: z.literal("restore"), items: z.array(z.object({ id: z.number().int().positive(), location: z.string().max(120).nullable(), warrantyUntil: dateOrBlank, endOfSupport: dateOrBlank })).min(1).max(MAX_BULK) }),
]);
inventoryBulkRouter.post("/", asyncHandler(async (req, res) => {
const parsed = inventoryBody.safeParse(req.body);
if (!parsed.success) return bad(res, parsed.error.issues[0]?.message ?? `Choose up to ${MAX_BULK} pieces of hardware and what to change.`);
const body = parsed.data;
const actor = req.currentUser!;
try {
if (body.action === "restore") {
const changed = await restoreAssets(body.items);
await recordAudit({ actor, category: "inventory", action: "bulk_change", detail: { change: "undo", changed } });
return res.json({ changed, skipped: [], undo: null });
}
const { ids, action: _a, ...change } = body;
const r = await changeAssets(ids, change);
await recordAudit({ actor, category: "inventory", action: "bulk_change", detail: { ...change, changed: r.changed, ids: detailIds(ids) } });
return res.json(r);
} catch (err) {
if (err instanceof BulkInputError) return bad(res, err.message);
throw err;
}
}));
+1 -1
View File
@@ -144,7 +144,7 @@ const updateSchema = z.object({
.partial()
.optional(),
healthChecks: z
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean() })
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20) })
.partial()
.optional(),
quietHours: z
+6
View File
@@ -94,6 +94,12 @@ export async function setAck(input: SetAckInput, now: Date = new Date()): Promis
return { ack: (await loadAck(input.alertId, now))!, before };
}
/** Puts an acknowledgement back exactly as it was (the undo of a bulk change): the same note, person, times and end. */
export async function restoreAck(alertId: string, previous: AckInfo, now: Date = new Date()): Promise<void> {
const values = { note: previous.note, assigneeUserId: previous.assignee?.id ?? null, assigneeName: previous.assignee?.name ?? null, ackedBy: previous.ackedBy, ackedAt: previous.ackedAt, until: previous.until, lastSeenAt: now.toISOString() };
await db.insert(alertAcks).values({ alertKey: alertId, ...values }).onConflictDoUpdate({ target: alertAcks.alertKey, set: values });
}
export async function clearAck(alertId: string): Promise<AckInfo | null> {
const before = await loadAck(alertId);
if (!before) return null;
+145
View File
@@ -0,0 +1,145 @@
/**
* The alert noise report: which alerts fire most often, how long they last, and which have been flapping — worked out from the incident record (every time an alert
* was raised and later went away is one incident, services/incidents.ts). It is for deciding what to quieten: raise a limit, ignore a service, fix the cause.
*
* An alert is the same alert each time it comes back (the incident record keys on the alert's own id), so "fired 14 times" is the same disk or the same check
* coming and going — which is the noise. Alerts that fired only once are left out: one is an event, not noise.
*/
import { and, gte } from "drizzle-orm";
import { db } from "../db/client.js";
import { alertIncidents } from "../db/schema.js";
import { flappingKeys, FLAP_WINDOW_MS, keyOfAlertId } from "./flapGuard.js";
import { getSettings } from "./settingsStore.js";
const DAY_MS = 86_400_000;
export const NOISE_DAYS = [7, 30, 90, 180] as const;
const MAX_ROWS = 50;
/** The threshold the report uses for "flapped" when flapping is switched off in Settings (so the figure still means something). */
const DEFAULT_FLAP_RAISES = 3;
export interface NoiseRow {
/** The alert's id, which is what the incident record keys on. */
alertId: string;
/** What it said the last time. */
message: string;
category: string;
source: string;
where: string;
/** How many times it was raised in the period. */
fired: number;
totalMinutes: number;
averageMinutes: number;
longestMinutes: number;
/** The most times it was raised within any one hour. */
mostInAnHour: number;
/** Raised enough times within an hour to count as flapping. */
flapped: boolean;
/** Flapping right now: its messages are held back. */
flappingNow: boolean;
openNow: boolean;
lastAt: string;
link: string | null;
/** What could be done about it, in a sentence. */
hint: string;
}
export interface NoiseReport {
days: number;
from: string;
to: string;
/** Every incident in the period, and how many different alerts they were. */
incidents: number;
distinctAlerts: number;
/** Alerts that fired more than once, which are what the table lists. */
repeating: number;
flapped: number;
/** All the time spent with an alert up, in the period, in minutes. */
totalMinutes: number;
byCategory: { category: string; incidents: number; repeating: number }[];
rows: NoiseRow[];
}
/** Pure: the most raises within any window of `windowMs` (the times are epoch ms). */
export function mostWithin(times: number[], windowMs: number): number {
const sorted = [...times].sort((a, b) => a - b);
let best = 0;
let from = 0;
for (let i = 0; i < sorted.length; i++) {
while (sorted[i] - sorted[from] >= windowMs) from++;
best = Math.max(best, i - from + 1);
}
return best;
}
/** What to try, from the kind of alert and the key it has. Plain advice; nothing it says is done for anyone. */
export function hintFor(category: string, alertId: string): string {
const key = keyOfAlertId(alertId, category);
if (key.startsWith("new-port:")) return "If this port should be listening, mark it as expected on the server's Ports card.";
if (key.startsWith("heartbeat")) return "If the job is just sometimes late, give its heartbeat more grace; if it fails, look at what it printed.";
if (key.startsWith("endpoint:")) return "Raise “Alert after … failures in a row” on the check, or its timeout or interval — or fix what is making it fail.";
if (key.startsWith("endpoint-cert:")) return "Renew the certificate, or lower how early the check warns.";
if (key.startsWith("failed-units:")) return "If a service is expected to fail, add it to “Services to ignore” on the server's Host health card; otherwise fix the service.";
if (key.startsWith("disk-forecast:") || category === "disk") return "Raise the limit for this disk (a server can have its own, per disk), or leave it alone on the server's page if it is meant to be full.";
if (category === "offline") return "If the server is just slow to report, raise “Server offline after” under Settings → Notifications, or give this server its own limit.";
if (category === "integration") return "Raise “Alert after … consecutive failures” under Settings → Notifications; the Call log (Integrations) shows what is failing.";
if (category === "backup") return "A guest that is backed up late each time may need its “too old” limit raised on its server, or its job moved.";
return "Acknowledge it with a note, silence it with maintenance while you work on it, or fix what raises it.";
}
export async function buildNoiseReport(days: number, now: Date = new Date()): Promise<NoiseReport> {
const span = (NOISE_DAYS as readonly number[]).includes(days) ? days : 30;
const from = new Date(now.getTime() - span * DAY_MS);
const rows = await db.select().from(alertIncidents).where(and(gte(alertIncidents.openedAt, from.toISOString())));
const threshold = (await getSettings()).healthChecks.flapRaises || DEFAULT_FLAP_RAISES;
const flapping = await flappingKeys().catch(() => new Set<string>());
const groups = new Map<string, typeof rows>();
for (const r of rows) groups.set(r.alertKey, [...(groups.get(r.alertKey) ?? []), r]);
let totalMinutes = 0;
const out: NoiseRow[] = [];
const categories = new Map<string, { incidents: number; repeating: number }>();
for (const [alertId, list] of groups) {
const minutes = list.map((r) => Math.max(1, Math.round(((r.resolvedAt ? Date.parse(r.resolvedAt) : now.getTime()) - Date.parse(r.openedAt)) / 60_000)));
const sum = minutes.reduce((n, m) => n + m, 0);
totalMinutes += sum;
const latest = [...list].sort((a, b) => b.openedAt.localeCompare(a.openedAt))[0];
const c = categories.get(latest.category) ?? { incidents: 0, repeating: 0 };
c.incidents += list.length;
if (list.length > 1) c.repeating++;
categories.set(latest.category, c);
if (list.length < 2) continue;
const most = mostWithin(list.map((r) => Date.parse(r.openedAt)), FLAP_WINDOW_MS);
out.push({
alertId,
message: latest.message,
category: latest.category,
source: latest.source,
where: latest.targetName ?? latest.source,
fired: list.length,
totalMinutes: sum,
averageMinutes: Math.round(sum / list.length),
longestMinutes: Math.max(...minutes),
mostInAnHour: most,
flapped: most >= threshold,
flappingNow: flapping.has(keyOfAlertId(alertId, latest.category)),
openNow: list.some((r) => r.resolvedAt === null),
lastAt: latest.openedAt,
link: latest.link,
hint: hintFor(latest.category, alertId),
});
}
out.sort((a, b) => b.fired - a.fired || b.totalMinutes - a.totalMinutes || a.message.localeCompare(b.message));
return {
days: span,
from: from.toISOString(),
to: now.toISOString(),
incidents: rows.length,
distinctAlerts: groups.size,
repeating: out.length,
flapped: out.filter((r) => r.flapped).length,
totalMinutes,
byCategory: [...categories].map(([category, v]) => ({ category, ...v })).sort((a, b) => b.incidents - a.incidents || a.category.localeCompare(b.category)),
rows: out.slice(0, MAX_ROWS),
};
}
+2
View File
@@ -23,6 +23,8 @@ export interface Alert {
causedBy?: { name: string; alertId?: string };
/** On a host's alert: the servers on it whose own alerts are folded under this one. */
affects?: string[];
/** Raised and cleared over and over: its raised and cleared messages are being held back (services/flapGuard.ts). */
flapping?: boolean;
/** What a maintenance window would have to be on to silence it, when there is such a thing — which is what "Silence" acts on. */
target: { type: "server" | "integration" | "dns_provider"; id: number; name: string } | null;
}
+5
View File
@@ -17,6 +17,7 @@ import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresho
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
import { recordFromReport } from "./incidents.js";
import { reconcileAcks } from "./alertAcks.js";
import { flappingKeys, keyOfAlertId } from "./flapGuard.js";
import { lastAttempt as lastSelfBackup, newestSnapshot, STALE_AFTER_HOURS as SELF_BACKUP_STALE_HOURS } from "./selfBackup.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
@@ -533,6 +534,10 @@ export async function collectAlerts(): Promise<AlertsReport> {
a.message.localeCompare(b.message),
);
// Those that keep coming and going are marked, so nobody mistakes the quiet of their held messages for the problem having gone.
const flapping = await flappingKeys().catch(() => new Set<string>());
if (flapping.size > 0) for (const a of alerts) if (flapping.has(keyOfAlertId(a.id, a.category))) a.flapping = true;
const active = alerts.filter((a) => !a.silenced);
const grouped = alerts.filter((a) => a.causedBy).length;
return {
+290
View File
@@ -0,0 +1,290 @@
/**
* Bulk actions: do the same thing to several alerts, servers or hardware entries at once. Each action says what it changed and returns, with it, the body that
* puts everything back — so "Undo" is only sending that body to the same address again, and an undo needs nothing remembered on the server. Whatever could not be
* done (an alert that has gone, a server already at its limit of tags, a change that would make a loop) is listed with the reason and does not stop the rest.
*/
import { and, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, maintenanceWindows, servers } from "../db/schema.js";
import { invalidateAlertsCache, getAlerts } from "./alerts.js";
import { clearAck, restoreAck, setAck, type AckInfo, type SetAckInput } from "./alertAcks.js";
import { impactOf } from "./dependencies.js";
import { isRealDate } from "./inventory.js";
import { listActiveWindows, startOrExtendWindow } from "./maintenance.js";
import { loadParentRows, wouldLoop } from "./serverParents.js";
import { InvalidTagError, normalizeTags, parseTags } from "./serverTags.js";
export const MAX_BULK = 200;
export interface Skipped {
id: string | number;
name: string;
reason: string;
}
export interface BulkResult<U> {
changed: number;
skipped: Skipped[];
/** The body that puts it back, or null when nothing changed. */
undo: U | null;
}
const unique = <T>(list: T[]): T[] => [...new Set(list)];
// ─── Maintenance (silence), shared by servers and alerts ───────────────────
export interface SilenceUndo {
action: "undo_silence";
/** Windows these made: ended again. */
created: number[];
/** Windows that were already there and were extended: put back as they were. */
restored: { id: number; endsAt: string; reason: string | null }[];
}
type Target = { type: "server" | "integration" | "dns_provider"; id: number };
/** Puts each target under maintenance (and, if asked, what depends on a server). Targets that are already under it have their window extended. */
export async function silenceTargets(targets: Target[], minutes: number, reason: string | null, createdBy: string, withDependents: boolean, now: Date = new Date()): Promise<{ silenced: { type: string; id: number; name: string }[]; skipped: Skipped[]; undo: SilenceUndo | null }> {
const wanted = new Map<string, Target & { why: string | null }>();
for (const t of targets) wanted.set(`${t.type}:${t.id}`, { ...t, why: reason });
if (withDependents) {
for (const t of targets) {
if (t.type !== "server" && t.type !== "integration") continue;
const impact = await impactOf(t.type, t.id, now);
for (const d of impact?.dependents ?? []) {
const key = `${d.type}:${d.id}`;
if (!wanted.has(key)) wanted.set(key, { type: d.type, id: d.id, why: `Depends on ${impact?.target.name ?? "a server in the same change"}${reason ? ` — ${reason}` : ""}`.slice(0, 200) });
}
}
}
const active = await listActiveWindows(now);
const created: number[] = [];
const restored: SilenceUndo["restored"] = [];
const silenced: { type: string; id: number; name: string }[] = [];
const skipped: Skipped[] = [];
for (const t of wanted.values()) {
const existing = active.find((w) => w.targetType === t.type && w.targetId === t.id);
const started = await startOrExtendWindow(t.type, t.id, minutes, t.why, createdBy, now);
if (!started) {
skipped.push({ id: t.id, name: `${t.type} #${t.id}`, reason: "it no longer exists" });
continue;
}
silenced.push({ type: t.type, id: t.id, name: started.target.name });
if (started.extended && existing) restored.push({ id: existing.id, endsAt: existing.endsAt, reason: existing.reason });
else created.push(started.window.id);
}
invalidateAlertsCache();
return { silenced, skipped, undo: silenced.length > 0 ? { action: "undo_silence", created, restored } : null };
}
export async function undoSilence(undo: Pick<SilenceUndo, "created" | "restored">, now: Date = new Date()): Promise<number> {
if (undo.created.length > 0) await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(inArray(maintenanceWindows.id, undo.created));
for (const r of undo.restored) await db.update(maintenanceWindows).set({ endsAt: r.endsAt, reason: r.reason }).where(eq(maintenanceWindows.id, r.id));
invalidateAlertsCache();
return undo.created.length + undo.restored.length;
}
// ─── Servers ───────────────────────────────────────────────────────────────
export interface TagsUndo {
action: "set_tags";
items: { id: number; tags: string[] }[];
}
/** Adds or takes away a tag on each server. A server that already has it (or lacks it), or has no room for another, is left as it was. */
export async function changeServerTag(ids: number[], change: { add: string } | { remove: string }): Promise<BulkResult<TagsUndo>> {
const rows = await db.select().from(servers).where(inArray(servers.id, unique(ids)));
const tag = normalizeTags(["add" in change ? change.add : change.remove])[0];
if (!tag) throw new InvalidTagError("A tag needs a name.");
const skipped: Skipped[] = unique(ids).filter((id) => !rows.some((r) => r.id === id)).map((id) => ({ id, name: `server #${id}`, reason: "it no longer exists" }));
const items: TagsUndo["items"] = [];
for (const r of rows) {
const before = parseTags(r.tags);
let after: string[];
if ("add" in change) {
if (before.includes(tag)) {
skipped.push({ id: r.id, name: r.name, reason: "it already has that tag" });
continue;
}
try {
after = normalizeTags([...before, tag]);
} catch (err) {
if (err instanceof InvalidTagError) {
skipped.push({ id: r.id, name: r.name, reason: err.message });
continue;
}
throw err;
}
} else {
if (!before.includes(tag)) {
skipped.push({ id: r.id, name: r.name, reason: "it doesn't have that tag" });
continue;
}
after = before.filter((t) => t !== tag);
}
await db.update(servers).set({ tags: after.length > 0 ? JSON.stringify(after) : null }).where(eq(servers.id, r.id));
items.push({ id: r.id, tags: before });
}
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "set_tags", items } : null };
}
/** Sets each server's tags to exactly what is given (the undo of a tag change). */
export async function setServerTags(items: { id: number; tags: string[] }[]): Promise<number> {
let n = 0;
for (const item of items) {
const tags = normalizeTags(item.tags);
const done = await db.update(servers).set({ tags: tags.length > 0 ? JSON.stringify(tags) : null }).where(eq(servers.id, item.id)).returning({ id: servers.id });
n += done.length;
}
return n;
}
export interface ParentUndo {
action: "restore_parents";
items: { id: number; parentServerId: number | null; noParent: boolean }[];
}
/** Sets which server each runs on. `parentServerId` null goes back to working it out from the Proxmox node; a change that would make a loop is skipped. */
export async function setServerParents(ids: number[], parentServerId: number | null): Promise<BulkResult<ParentUndo>> {
const rows = await loadParentRows();
const skipped: Skipped[] = [];
const items: ParentUndo["items"] = [];
if (parentServerId !== null && !rows.some((r) => r.id === parentServerId)) throw new RangeError("That server doesn't exist.");
for (const id of unique(ids)) {
const row = rows.find((r) => r.id === id);
if (!row) {
skipped.push({ id, name: `server #${id}`, reason: "it no longer exists" });
continue;
}
if (parentServerId !== null) {
if (parentServerId === id) {
skipped.push({ id, name: row.name, reason: "a server can't run on itself" });
continue;
}
if (wouldLoop(rows, id, parentServerId)) {
skipped.push({ id, name: row.name, reason: "that server already runs on this one, so each would be waiting for the other" });
continue;
}
}
items.push({ id, parentServerId: row.parentServerId ?? null, noParent: row.noParent ?? false });
await db.update(servers).set({ parentServerId, noParent: false }).where(eq(servers.id, id));
row.parentServerId = parentServerId;
row.noParent = false;
}
if (items.length > 0) invalidateAlertsCache();
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_parents", items } : null };
}
export async function restoreServerParents(items: ParentUndo["items"]): Promise<number> {
let n = 0;
for (const i of items) {
const done = await db.update(servers).set({ parentServerId: i.parentServerId, noParent: i.noParent }).where(eq(servers.id, i.id)).returning({ id: servers.id });
n += done.length;
}
if (n > 0) invalidateAlertsCache();
return n;
}
// ─── Alerts ────────────────────────────────────────────────────────────────
export interface AcksUndo {
action: "restore_acks";
items: { alertId: string; previous: AckInfo | null }[];
}
/** Acknowledges each alert the same way. Alerts that are no longer there are skipped. */
export async function acknowledgeAlerts(alertIds: string[], input: Omit<SetAckInput, "alertId">): Promise<BulkResult<AcksUndo>> {
const present = new Map((await getAlerts(false)).alerts.map((a) => [a.id, a]));
const skipped: Skipped[] = [];
const items: AcksUndo["items"] = [];
for (const alertId of unique(alertIds)) {
if (!present.has(alertId)) {
skipped.push({ id: alertId, name: alertId, reason: "it isn't there any more" });
continue;
}
const { before } = await setAck({ ...input, alertId });
items.push({ alertId, previous: before });
}
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_acks", items } : null };
}
/** Takes the acknowledgement off each alert that has one. */
export async function unacknowledgeAlerts(alertIds: string[]): Promise<BulkResult<AcksUndo>> {
const skipped: Skipped[] = [];
const items: AcksUndo["items"] = [];
for (const alertId of unique(alertIds)) {
const before = await clearAck(alertId);
if (!before) {
skipped.push({ id: alertId, name: alertId, reason: "nobody had acknowledged it" });
continue;
}
items.push({ alertId, previous: before });
}
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_acks", items } : null };
}
/** Puts each alert's acknowledgement back as it was (or takes it off, if it had none). */
export async function restoreAcks(items: AcksUndo["items"]): Promise<number> {
for (const i of items) {
if (i.previous === null) await clearAck(i.alertId);
else await restoreAck(i.alertId, i.previous);
}
return items.length;
}
/** The maintenance targets of the alerts given (those with none can't be silenced). */
export async function targetsOfAlerts(alertIds: string[]): Promise<{ targets: Target[]; skipped: Skipped[] }> {
const present = new Map((await getAlerts(false)).alerts.map((a) => [a.id, a]));
const targets = new Map<string, Target>();
const skipped: Skipped[] = [];
for (const alertId of unique(alertIds)) {
const alert = present.get(alertId);
if (!alert) skipped.push({ id: alertId, name: alertId, reason: "it isn't there any more" });
else if (!alert.target) skipped.push({ id: alertId, name: alert.message.slice(0, 80), reason: "there is nothing to put under maintenance for it" });
else targets.set(`${alert.target.type}:${alert.target.id}`, { type: alert.target.type, id: alert.target.id });
}
return { targets: [...targets.values()], skipped };
}
// ─── Hardware ──────────────────────────────────────────────────────────────
export interface AssetsUndo {
action: "restore";
items: { id: number; location: string | null; warrantyUntil: string | null; endOfSupport: string | null }[];
}
export class BulkInputError extends Error {}
/** What a blank in a form means: nothing, not an empty string. */
const blank = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
/** Sets the location and/or the dates of each piece of hardware. Whatever is not given is left as it is; a blank clears. */
export async function changeAssets(ids: number[], change: { location?: string | null; warrantyUntil?: string | null; endOfSupport?: string | null }): Promise<BulkResult<AssetsUndo>> {
const set: Partial<typeof assets.$inferInsert> = {};
if (change.location !== undefined) set.location = blank(change.location);
for (const field of ["warrantyUntil", "endOfSupport"] as const) {
if (change[field] === undefined) continue;
const v = blank(change[field]);
if (v !== null && !isRealDate(v)) throw new BulkInputError("Use a real date, like 2027-06-30.");
set[field] = v;
}
if (Object.keys(set).length === 0) throw new BulkInputError("Say what to change.");
const rows = await db.select().from(assets).where(inArray(assets.id, unique(ids)));
const skipped: Skipped[] = unique(ids).filter((id) => !rows.some((r) => r.id === id)).map((id) => ({ id, name: `hardware #${id}`, reason: "it no longer exists" }));
const items: AssetsUndo["items"] = [];
for (const r of rows) {
items.push({ id: r.id, location: r.location, warrantyUntil: r.warrantyUntil, endOfSupport: r.endOfSupport });
await db.update(assets).set({ ...set, updatedAt: new Date().toISOString() }).where(eq(assets.id, r.id));
}
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore", items } : null };
}
export async function restoreAssets(items: AssetsUndo["items"]): Promise<number> {
let n = 0;
for (const i of items) {
const done = await db.update(assets).set({ location: i.location, warrantyUntil: i.warrantyUntil, endOfSupport: i.endOfSupport, updatedAt: new Date().toISOString() }).where(and(eq(assets.id, i.id))).returning({ id: assets.id });
n += done.length;
}
return n;
}
+22
View File
@@ -157,6 +157,28 @@ export function describeChange(category: string, action: string, detail: Record<
return { title: `Endpoint check changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
case "endpoint/delete":
return { title: `Endpoint check removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "server/bulk_tag": {
const n = typeof detail.changed === "number" ? detail.changed : null;
return { title: detail.change === "undo" ? "A tag change on several servers was undone" : `${str(detail.change) === "remove_tag" ? "Tag taken off" : "Tag added to"} ${n ?? "several"} server${n === 1 ? "" : "s"}${str(detail.tag) ? `: ${str(detail.tag)}` : ""}`, tone: "neutral", detail: null };
}
case "server/bulk_runs_on":
return { title: detail.change === "undo" ? "A change of which server others run on was undone" : `Which server ${typeof detail.changed === "number" ? detail.changed : "several"} run on was changed`, tone: "neutral", detail: null };
case "maintenance/bulk_start":
return { title: `Maintenance started on several at once${Array.isArray(detail.servers ?? detail.targets) ? `: ${((detail.servers ?? detail.targets) as unknown[]).slice(0, 4).join(", ")}` : ""}`, tone: "neutral", detail: typeof detail.reason === "string" ? detail.reason : null };
case "maintenance/bulk_undo":
return { title: "Maintenance started on several at once was undone", tone: "neutral", detail: null };
case "alert/bulk_ack":
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} alert${detail.changed === 1 ? "" : "s"} acknowledged at once`, tone: "neutral", detail: typeof detail.note === "string" && detail.note ? detail.note : null };
case "alert/bulk_unack":
return { title: `Acknowledgement taken off ${typeof detail.changed === "number" ? detail.changed : "several"} alert${detail.changed === 1 ? "" : "s"}`, tone: "neutral", detail: null };
case "alert/bulk_undo":
return { title: "Acknowledging several alerts at once was undone", tone: "neutral", detail: null };
case "inventory/bulk_change":
return { title: detail.change === "undo" ? "A change to several pieces of hardware was undone" : `${typeof detail.changed === "number" ? detail.changed : "Several"} piece${detail.changed === 1 ? "" : "s"} of hardware changed at once`, tone: "neutral", detail: genericSummary(detail, ["ids", "changed", "change"]) };
case "alert/flapping_started":
return { title: `Alert began flapping${str(detail.message) ? `: ${str(detail.message)}` : ""}`, tone: "bad", detail: typeof detail.raises === "number" ? `raised ${detail.raises} times within an hour; its messages are held back` : null };
case "alert/flapping_ended":
return { title: `Alert settled${str(detail.message) ? `: ${str(detail.message)}` : ""}`, tone: detail.active === true ? "bad" : "good", detail: detail.active === true ? "steady for an hour, and still active" : "steady for an hour, and cleared" };
case "report/send_monthly":
return { title: `Monthly report sent${str(detail.month) ? ` (${str(detail.month)})` : ""}`, tone: "neutral", detail: null };
case "patch_plan/create":
+161
View File
@@ -0,0 +1,161 @@
/**
* Flapping: an alert that is raised and clears over and over — a service that crashes and restarts, a link that drops every few minutes — would send a raised and a
* recovered message each time, which is noise that hides the one thing worth knowing: that it is unstable. So when the same alert has been raised several times within
* an hour, it is called *flapping*: one message says so, the Alerts page marks it, and its further raised and recovered messages are held back. When it has been quiet
* for an hour it has settled, and one message says what state it is in.
*
* It sits in the notification layer (notifyHealthIssues / notifyHealthRecovered), so it covers everything that goes through the health monitor's bookkeeping: servers,
* disks, services, new ports, endpoint checks and heartbeats. The alert itself is not touched — it is still raised, counted, and in the incident record — only what is
* *sent* is calmer. The event webhooks, which are for machines, still hear every raise and clear.
*/
import { recordAudit } from "./audit.js";
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
const FLAG = "flapState";
/** The raises that count are those within this long. */
export const FLAP_WINDOW_MS = 60 * 60_000;
/** A flapping alert that has had no raise or clear for this long has settled. */
export const SETTLE_MS = 60 * 60_000;
export interface FlapEntry {
/** When it was raised (and announced), epoch ms, within the window. */
raises: number[];
flapping: boolean;
/** When it was called flapping. */
since: number | null;
/** The last time it was raised or cleared while flapping. */
lastEvent: number;
/** What it says, for the messages that come later. */
message: string;
}
export type FlapState = Record<string, FlapEntry>;
export type RaiseVerdict = "pass" | "started" | "held";
/** Pure: what a raise does to an alert's record — and whether its message goes out, becomes the one "flapping" message, or is held. A threshold of 0 turns it off. */
export function registerRaise(entry: FlapEntry | undefined, now: number, threshold: number, message: string): { entry: FlapEntry; verdict: RaiseVerdict } {
const kept = (entry?.raises ?? []).filter((t) => now - t < FLAP_WINDOW_MS);
if (entry?.flapping) return { entry: { ...entry, raises: [...kept, now], lastEvent: now, message }, verdict: "held" };
const raises = [...kept, now];
if (threshold > 0 && raises.length >= threshold) return { entry: { raises, flapping: true, since: now, lastEvent: now, message }, verdict: "started" };
return { entry: { raises, flapping: false, since: null, lastEvent: now, message }, verdict: "pass" };
}
/** Pure: whether a recovery message is held (the alert is flapping), and the record after it. */
export function registerRecovery(entry: FlapEntry | undefined, now: number): { entry: FlapEntry | undefined; hold: boolean } {
if (!entry?.flapping) return { entry, hold: false };
return { entry: { ...entry, lastEvent: now }, hold: true };
}
/** Pure: whether a flapping alert has been quiet long enough to have settled. */
export const hasSettled = (entry: FlapEntry, now: number) => entry.flapping && now - entry.lastEvent >= SETTLE_MS;
// ─── What is kept ──────────────────────────────────────────────────────────
async function load(): Promise<FlapState> {
const raw = await getInternalFlag(FLAG);
if (!raw) return {};
try {
const v = JSON.parse(raw);
return v && typeof v === "object" ? (v as FlapState) : {};
} catch {
return {};
}
}
async function save(state: FlapState, now: number): Promise<void> {
// An alert that is not flapping and has had no raise within the window has nothing worth remembering.
const next: FlapState = {};
for (const [key, e] of Object.entries(state)) if (e.flapping || e.raises.some((t) => now - t < FLAP_WINDOW_MS)) next[key] = e;
await setInternalFlag(FLAG, JSON.stringify(next));
}
// Two monitors can announce at the same moment; each reads the record, changes it and writes it back, so they take turns.
let turn: Promise<unknown> = Promise.resolve();
function inTurn<T>(work: () => Promise<T>): Promise<T> {
const result = turn.then(work, work);
turn = result.catch(() => undefined);
return result;
}
const thresholdNow = async () => (await getSettings()).healthChecks.flapRaises;
export interface Keyed {
key?: string;
message: string;
}
/**
* Looks at alerts that have just been raised: those that go out as they are, and those that have just begun flapping (each of which goes out once, as the
* "flapping" message, with how many times it has been raised). Held ones are in neither list.
*/
export async function dampRaised<T extends Keyed>(issues: T[], now: Date = new Date()): Promise<{ pass: T[]; started: (T & { count: number })[] }> {
const withKey = issues.filter((i) => i.key);
if (withKey.length === 0) return { pass: issues, started: [] };
return inTurn(async () => {
const threshold = await thresholdNow();
const state = await load();
const pass: T[] = [];
const started: (T & { count: number })[] = [];
for (const issue of issues) {
if (!issue.key) {
pass.push(issue);
continue;
}
const { entry, verdict } = registerRaise(state[issue.key], now.getTime(), threshold, issue.message);
state[issue.key] = entry;
if (verdict === "pass") pass.push(issue);
else if (verdict === "started") {
started.push({ ...issue, count: entry.raises.length });
await recordAudit({ category: "alert", action: "flapping_started", detail: { key: issue.key, message: issue.message, raises: entry.raises.length } }).catch(() => undefined);
}
}
await save(state, now.getTime());
return { pass, started };
});
}
/** Looks at alerts that have just cleared: those of flapping alerts are held (their moment is noted, so the quiet is counted from it). */
export async function dampRecovered<T extends { key?: string }>(resolved: T[], now: Date = new Date()): Promise<T[]> {
if (!resolved.some((r) => r.key)) return resolved;
return inTurn(async () => {
const state = await load();
const pass: T[] = [];
for (const r of resolved) {
if (!r.key) {
pass.push(r);
continue;
}
const { entry, hold } = registerRecovery(state[r.key], now.getTime());
if (entry) state[r.key] = entry;
if (!hold) pass.push(r);
}
await save(state, now.getTime());
return pass;
});
}
/** The alerts that have been quiet for an hour and so have settled — dropped from the record, with whether each is active now. */
export async function settleFlaps(activeKeys: Set<string>, now: Date = new Date()): Promise<{ key: string; message: string; active: boolean; raises: number }[]> {
return inTurn(async () => {
const state = await load();
const settled: { key: string; message: string; active: boolean; raises: number }[] = [];
for (const [key, e] of Object.entries(state)) {
if (!hasSettled(e, now.getTime())) continue;
settled.push({ key, message: e.message, active: activeKeys.has(key), raises: e.raises.length });
delete state[key];
await recordAudit({ category: "alert", action: "flapping_ended", detail: { key, message: e.message, active: activeKeys.has(key) } }).catch(() => undefined);
}
if (settled.length > 0) await save(state, now.getTime());
return settled;
});
}
/** The alerts that are flapping now (by the key the health checks give them). */
export async function flappingKeys(): Promise<Set<string>> {
const state = await load();
return new Set(Object.entries(state).filter(([, e]) => e.flapping).map(([key]) => key));
}
/** The key a health condition has behind an alert's id ("disk:" + "disk:server:3:/" is the id of the alert about "disk:server:3:/"). */
export const keyOfAlertId = (id: string, category: string) => (id.startsWith(`${category}:`) ? id.slice(category.length + 1) : id);
+23
View File
@@ -0,0 +1,23 @@
/**
* Every few minutes: flapping alerts that have been quiet for an hour have settled — say so once, with the state each is in now (see services/flapGuard.ts).
*/
import { getAlerts } from "./alerts.js";
import { keyOfAlertId, settleFlaps } from "./flapGuard.js";
import { notifySettled } from "./notify.js";
/** One look: returns how many settled. */
export async function settleFlappingAlerts(now: Date = new Date()): Promise<number> {
const report = await getAlerts(false);
const active = new Set(report.alerts.map((a) => keyOfAlertId(a.id, a.category)));
const settled = await settleFlaps(active, now);
if (settled.length > 0) await notifySettled(settled);
return settled.length;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initFlapMonitor(): void {
if (timer) return;
timer = setInterval(() => void settleFlappingAlerts().catch((err) => console.error("[flap] couldn't look at the flapping alerts:", err)), 5 * 60_000);
timer.unref();
}
+27
View File
@@ -7,6 +7,7 @@ import { db } from "../db/client.js";
import { inArray } from "drizzle-orm";
import { pushSubscriptions, servers } from "../db/schema.js";
import { parseTags } from "./serverTags.js";
import { dampRaised, dampRecovered } from "./flapGuard.js";
import { firstMatchingRule, metaOfConditionKey, pushAccepts, type ChannelId, type NotifyMeta } from "./notifyRouting.js";
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
@@ -483,7 +484,12 @@ export async function notifyHealthIssues(issues: { message: string; key?: string
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
// An alert that keeps coming and going is announced once as "flapping" and then left quiet (services/flapGuard.ts); that is tracked whether or not this is switched on.
const { pass, started } = await dampRaised(issues);
if (!(await eventEnabled("healthAlerts"))) return;
if (started.length > 0) await notifyFlapping(started);
if (pass.length === 0) return;
issues = pass;
await notifyGrouped(issues, issueMeta, (group) => ({
title: "Homelab Manager — Health Alert",
message: `${group.length} new issue${group.length !== 1 ? "s" : ""}:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`,
@@ -493,13 +499,34 @@ export async function notifyHealthIssues(issues: { message: string; key?: string
export async function notifyHealthRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
const passed = await dampRecovered(resolved);
if (!(await eventEnabled("healthAlerts"))) return;
if (passed.length === 0) return;
resolved = passed;
await notifyGrouped(resolved, issueMeta, (group) => ({
title: "Homelab Manager — Health Recovered",
message: `${group.length} issue${group.length !== 1 ? "s" : ""} cleared:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
}));
}
/** An alert that has just been called flapping: said once, with how often it has come, instead of each time. */
export async function notifyFlapping(items: { message: string; key?: string; severity?: string; count: number }[]): Promise<void> {
if (items.length === 0) return;
await notifyGrouped(items, issueMeta, (group) => ({
title: "Homelab Manager — Flapping alert",
message: `${group.length === 1 ? "An alert keeps" : `${group.length} alerts keep`} coming and going:\n\n${group.map((i) => `↻ ${i.message} — raised ${i.count} times within an hour`).join("\n")}\n\nIts further messages are held back until it has been quiet for an hour; it stays on the Alerts page, marked as flapping.`,
}));
}
/** Flapping alerts that have been quiet for an hour: what state each is in now. */
export async function notifySettled(items: { message: string; key: string; active: boolean }[]): Promise<void> {
if (items.length === 0 || !(await eventEnabled("healthAlerts"))) return;
await notifyGrouped(items, issueMeta, (group) => ({
title: "Homelab Manager — Alert settled",
message: `${group.length === 1 ? "An alert that was flapping has" : `${group.length} alerts that were flapping have`} been steady for an hour:\n\n${group.map((i) => `${i.active ? "⚠ still active" : "✓ cleared"} — ${i.message}`).join("\n")}`,
}));
}
export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
+3 -1
View File
@@ -115,6 +115,8 @@ export interface HealthCheckSettings {
restoreTestDays: number;
/** A server that starts listening on a TCP port it did not at first, and that is not noted on its Ports card, is a warning (services/newPorts.ts). */
newPortAlerts: boolean;
/** An alert raised this many times within an hour is "flapping": one message says so and its further raised/cleared messages are held until it has been quiet for an hour. 0 turns it off. */
flapRaises: number;
}
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
@@ -285,7 +287,7 @@ const DEFAULTS: AppSettings = {
selfBackup: { enabled: true, time: "03:30", keep: 14 },
profilePictures: { gravatar: false, identityProvider: false },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3 },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
consistency: { excludedRanges: ["172.16.0.0/12"] },
+346
View File
@@ -0,0 +1,346 @@
/**
* Bulk actions: the same thing done to several alerts, servers or hardware entries, who may do which, what is skipped and why, and that the body each one hands back
* puts everything exactly as it was.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-bulk-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, heartbeats, assets, alertAcks, maintenanceWindows, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { serversBulkRouter, alertsBulkRouter, inventoryBulkRouter } = await import("../src/routes/bulk.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { listActiveWindows } = await import("../src/services/maintenance.js");
const { getChangeLog } = await import("../src/services/changeLog.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
let app: Server;
let base = "";
let who = "operator";
const send = async (path: string, body: unknown) => {
const res = await fetch(`${base}${path}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
return { status: res.status, body: (await res.json().catch(() => null)) as any };
};
const tagsOf = async (id: number) => JSON.parse((await db.select().from(servers).where(eq(servers.id, id)))[0].tags ?? "[]") as string[];
const ids: Record<string, number> = {};
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/servers/bulk", serversBulkRouter);
a.use("/api/alerts/bulk", alertsBulkRouter);
a.use("/api/inventory/bulk", inventoryBulkRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
for (const [name, tags] of [["web", ["lab"]], ["api", ["lab", "prod"]], ["db", null], ["host", null]] as const) {
ids[name] = (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, tags: tags ? JSON.stringify(tags) : null, lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
}
// a guest that runs on host: what depends on it
ids.guest = (await db.insert(servers).values({ name: "guest", apiTokenHash: "g", apiTokenPrefix: "g", lastSeenAt: new Date().toISOString(), parentServerId: ids.host }).returning({ id: servers.id }))[0].id;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
describe("servers: tags", () => {
test("a tag is added to those that lack it; those that have it and those that have gone are listed with the reason; undo puts the tags back exactly", async () => {
const r = await send("/servers/bulk", { action: "add_tag", ids: [ids.web, ids.api, ids.db, 99999], tag: "Nightly" });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 3);
assert.deepEqual(await tagsOf(ids.web), ["lab", "nightly"], "written the way tags are");
assert.deepEqual(await tagsOf(ids.db), ["nightly"]);
assert.deepEqual(r.body.undo.items.map((i: { id: number; tags: string[] }) => [i.id, i.tags]).sort(), [[ids.web, ["lab"]], [ids.api, ["lab", "prod"]], [ids.db, []]].sort());
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it no longer exists"]);
const again = await send("/servers/bulk", { action: "add_tag", ids: [ids.web, ids.db], tag: "nightly" });
assert.equal(again.body.changed, 0);
assert.equal(again.body.undo, null, "nothing changed, so nothing to undo");
assert.deepEqual(again.body.skipped.map((s: { reason: string }) => s.reason), ["it already has that tag", "it already has that tag"]);
const undone = await send("/servers/bulk", r.body.undo);
assert.equal(undone.body.changed, 3);
assert.deepEqual([await tagsOf(ids.web), await tagsOf(ids.api), await tagsOf(ids.db)], [["lab"], ["lab", "prod"], []]);
assert.equal((await db.select().from(servers).where(eq(servers.id, ids.db)))[0].tags, null, "no tags is none, not an empty list");
});
test("a tag is taken off those that have it", async () => {
const r = await send("/servers/bulk", { action: "remove_tag", ids: [ids.web, ids.api, ids.db], tag: "lab" });
assert.equal(r.body.changed, 2);
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it doesn't have that tag"]);
assert.deepEqual([await tagsOf(ids.web), await tagsOf(ids.api)], [[], ["prod"]]);
await send("/servers/bulk", r.body.undo);
assert.deepEqual([await tagsOf(ids.web), await tagsOf(ids.api)], [["lab"], ["lab", "prod"]]);
});
test("a wrong tag is refused with the reason; a server with no room for another is skipped; the rest are done", async () => {
assert.equal((await send("/servers/bulk", { action: "add_tag", ids: [ids.web], tag: "bad tag!" })).status, 400);
const full = Array.from({ length: 12 }, (_, i) => `t${String(i).padStart(2, "0")}`);
await db.update(servers).set({ tags: JSON.stringify(full) }).where(eq(servers.id, ids.host));
const r = await send("/servers/bulk", { action: "add_tag", ids: [ids.host, ids.db], tag: "extra" });
assert.equal(r.body.changed, 1);
assert.match(r.body.skipped[0].reason, /at most 12 tags/);
await send("/servers/bulk", r.body.undo);
await db.update(servers).set({ tags: null }).where(eq(servers.id, ids.host));
});
});
describe("servers: maintenance", () => {
test("each is put under maintenance, one already under it is extended; undo ends the new and puts the old back as it was", async () => {
const T = new Date();
await db.delete(maintenanceWindows);
const before = (await db.insert(maintenanceWindows).values({ targetType: "server", targetId: ids.db, reason: "earlier", startedAt: new Date(T.getTime() - 60_000).toISOString(), endsAt: new Date(T.getTime() + 10 * 60_000).toISOString() }).returning())[0];
const r = await send("/servers/bulk", { action: "silence", ids: [ids.web, ids.db], minutes: 90, reason: "Patching" });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 2);
assert.deepEqual(r.body.silenced.map((s: { name: string }) => s.name).sort(), ["db", "web"]);
const during = await listActiveWindows();
assert.equal(during.some((w) => w.targetId === ids.web), true);
assert.ok(Date.parse(during.find((w) => w.targetId === ids.db)!.endsAt) > T.getTime() + 60 * 60_000, "extended to ninety minutes");
assert.equal(r.body.undo.created.length, 1);
assert.deepEqual(r.body.undo.restored.map((x: { id: number; endsAt: string }) => [x.id, x.endsAt]), [[before.id, before.endsAt]]);
const undone = await send("/servers/bulk", r.body.undo);
assert.equal(undone.body.changed, 2);
const after = await listActiveWindows();
assert.equal(after.some((w) => w.targetId === ids.web), false, "the one it made is over");
const old = (await db.select().from(maintenanceWindows).where(eq(maintenanceWindows.id, before.id)))[0];
assert.deepEqual([old.endsAt, old.reason], [before.endsAt, "earlier"], "the one that was there is as it was");
});
test("what depends on a server can be silenced with it", async () => {
await db.delete(maintenanceWindows);
const r = await send("/servers/bulk", { action: "silence", ids: [ids.host], minutes: 30, withDependents: true });
assert.deepEqual(r.body.silenced.map((s: { name: string }) => s.name).sort(), ["guest", "host"]);
assert.match((await listActiveWindows()).find((w) => w.targetId === ids.guest)!.reason ?? "", /^Depends on host/);
await send("/servers/bulk", r.body.undo);
assert.equal((await listActiveWindows()).length, 0);
});
test("how long must be sensible", async () => {
assert.equal((await send("/servers/bulk", { action: "silence", ids: [ids.web], minutes: 1 })).status, 400);
assert.equal((await send("/servers/bulk", { action: "silence", ids: [ids.web], minutes: 99999 })).status, 400);
});
});
describe("servers: which server another runs on", () => {
test("an administrator sets it for several; a loop is skipped; undo restores each exactly; an operator may not", async () => {
who = "operator";
assert.equal((await send("/servers/bulk", { action: "set_parent", ids: [ids.web], parentServerId: ids.host })).status, 403);
who = "admin";
const r = await send("/servers/bulk", { action: "set_parent", ids: [ids.web, ids.api, ids.host], parentServerId: ids.host });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 2);
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["a server can't run on itself"]);
const parent = async (id: number) => (await db.select().from(servers).where(eq(servers.id, id)))[0].parentServerId;
assert.deepEqual([await parent(ids.web), await parent(ids.api)], [ids.host, ids.host]);
// guest already runs on host: making host run on guest would close a loop
const loop = await send("/servers/bulk", { action: "set_parent", ids: [ids.host], parentServerId: ids.guest });
assert.equal(loop.body.changed, 0);
assert.match(loop.body.skipped[0].reason, /each would be waiting for the other/);
assert.equal((await send("/servers/bulk", { action: "set_parent", ids: [ids.web], parentServerId: 99999 })).status, 400);
await send("/servers/bulk", r.body.undo);
assert.deepEqual([await parent(ids.web), await parent(ids.api)], [null, null]);
const auto = await send("/servers/bulk", { action: "set_parent", ids: [ids.web], parentServerId: null });
assert.equal(auto.body.changed, 1, "null goes back to working it out");
who = "operator";
});
test("a viewer can do none of it", async () => {
who = "viewer";
assert.equal((await send("/servers/bulk", { action: "add_tag", ids: [ids.web], tag: "x" })).status, 403);
assert.equal((await send("/servers/bulk", { action: "silence", ids: [ids.web], minutes: 30 })).status, 403);
who = "operator";
});
test("too many, none, or something that isn't an action are refused", async () => {
assert.equal((await send("/servers/bulk", { action: "add_tag", ids: [], tag: "x" })).status, 400);
assert.equal((await send("/servers/bulk", { action: "add_tag", ids: Array.from({ length: 201 }, (_, i) => i + 1), tag: "x" })).status, 400);
assert.equal((await send("/servers/bulk", { action: "delete_everything", ids: [1] })).status, 400);
});
});
describe("alerts", () => {
const alertIds: string[] = [];
before(async () => {
await db.delete(maintenanceWindows);
// three late heartbeats: two on servers (so each has something to silence, and both on the same one), one on nothing
const old = "2026-01-01 00:00:00";
const mk = async (name: string, serverId: number | null) => (await db.insert(heartbeats).values({ name, token: `hb_${name}`.padEnd(24, "x"), periodMinutes: 60, graceMinutes: 5, serverId, createdAt: old }).returning())[0];
const h1 = await mk("job1", ids.web);
const h2 = await mk("job2", ids.web);
const h3 = await mk("job3", null);
invalidateAlertsCache();
const report = await getAlerts(true);
for (const h of [h1, h2, h3]) alertIds.push(report.alerts.find((a) => a.message.startsWith(h.name))!.id);
});
test("several are acknowledged the same way; one that has gone is skipped; undo takes them off again", async () => {
const r = await send("/alerts/bulk", { action: "acknowledge", alertIds: [...alertIds, "automation:heartbeat:99999"], note: "Known, fixing tonight", hours: 12 });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 3);
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it isn't there any more"]);
const rows = await db.select().from(alertAcks);
assert.equal(rows.length, 3);
assert.ok(rows.every((x) => x.note === "Known, fixing tonight" && x.ackedBy === "Olga" && x.until !== null));
assert.ok(r.body.undo.items.every((i: { previous: unknown }) => i.previous === null));
await send("/alerts/bulk", r.body.undo);
assert.equal((await db.select().from(alertAcks)).length, 0);
});
test("an earlier acknowledgement is put back exactly — its note, who it was handed to and when it ends", async () => {
await send("/alerts/bulk", { action: "acknowledge", alertIds: [alertIds[0]], note: "first", hours: 5 });
const [first] = await db.select().from(alertAcks);
const over = await send("/alerts/bulk", { action: "acknowledge", alertIds: [alertIds[0], alertIds[1]], note: "second" });
assert.equal(over.body.changed, 2);
assert.equal((await db.select().from(alertAcks).where(eq(alertAcks.alertKey, alertIds[0])))[0].note, "second");
await send("/alerts/bulk", over.body.undo);
const rows = await db.select().from(alertAcks);
assert.equal(rows.length, 1, "the second alert had none before");
assert.deepEqual([rows[0].note, rows[0].ackedAt, rows[0].until, rows[0].ackedBy], [first.note, first.ackedAt, first.until, first.ackedBy]);
});
test("acknowledgements are taken off, those with none are skipped, and undo puts them back", async () => {
await db.delete(alertAcks);
await send("/alerts/bulk", { action: "acknowledge", alertIds: [alertIds[0], alertIds[1]], note: "mine" });
const r = await send("/alerts/bulk", { action: "unacknowledge", alertIds });
assert.equal(r.body.changed, 2);
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["nobody had acknowledged it"]);
assert.equal((await db.select().from(alertAcks)).length, 0);
await send("/alerts/bulk", r.body.undo);
assert.deepEqual((await db.select().from(alertAcks)).map((x) => x.note), ["mine", "mine"]);
await db.delete(alertAcks);
});
test("handing them to someone who cannot act is refused, and nothing changes", async () => {
const r = await send("/alerts/bulk", { action: "acknowledge", alertIds, assigneeId: (await db.select().from(users).where(eq(users.oidcSub, "viewer")))[0].id });
assert.equal(r.status, 400);
assert.equal((await db.select().from(alertAcks)).length, 0);
});
test("silencing puts each distinct target under maintenance once; an alert with nothing to silence is skipped; undo ends it", async () => {
await db.delete(maintenanceWindows);
const r = await send("/alerts/bulk", { action: "silence", alertIds, minutes: 60, reason: "Weekend" });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 1, "two alerts, one server");
assert.equal(r.body.silenced[0].name, "web");
assert.equal(r.body.skipped.length, 1);
assert.match(r.body.skipped[0].reason, /nothing to put under maintenance/);
assert.equal((await listActiveWindows()).length, 1);
await send("/alerts/bulk", r.body.undo);
assert.equal((await listActiveWindows()).length, 0);
});
test("an operator is needed; a viewer is not enough", async () => {
who = "viewer";
assert.equal((await send("/alerts/bulk", { action: "acknowledge", alertIds })).status, 403);
who = "operator";
});
});
describe("hardware", () => {
const hw: Record<string, number> = {};
const row = async (id: number) => (await db.select().from(assets).where(eq(assets.id, id)))[0];
before(async () => {
const now = new Date().toISOString();
for (const [name, location, warrantyUntil] of [["NAS", "Rack A", "2027-01-01"], ["Switch", null, null], ["UPS", "Hall", "2026-12-01"]] as const) {
hw[name] = (await db.insert(assets).values({ name, kind: "other", location, warrantyUntil, createdAt: now, updatedAt: now } as never).returning({ id: assets.id }))[0].id;
}
});
test("a location is set on several at once, a blank clears it, and undo puts each one's own back", async () => {
const r = await send("/inventory/bulk", { action: "change", ids: [hw.NAS, hw.Switch, hw.UPS, 99999], location: "Closet" });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 3);
assert.deepEqual(r.body.skipped.map((s: { reason: string }) => s.reason), ["it no longer exists"]);
assert.deepEqual([(await row(hw.NAS)).location, (await row(hw.Switch)).location], ["Closet", "Closet"]);
await send("/inventory/bulk", r.body.undo);
assert.deepEqual([(await row(hw.NAS)).location, (await row(hw.Switch)).location, (await row(hw.UPS)).location], ["Rack A", null, "Hall"]);
const cleared = await send("/inventory/bulk", { action: "change", ids: [hw.NAS], location: " " });
assert.equal((await row(hw.NAS)).location, null);
await send("/inventory/bulk", cleared.body.undo);
assert.equal((await row(hw.NAS)).location, "Rack A");
});
test("the end of support and the warranty are set together or one at a time; whatever is not named is left alone; undo restores", async () => {
const r = await send("/inventory/bulk", { action: "change", ids: [hw.NAS, hw.Switch], endOfSupport: "2029-06-30" });
assert.equal(r.body.changed, 2);
const nas = await row(hw.NAS);
assert.deepEqual([nas.endOfSupport, nas.warrantyUntil, nas.location], ["2029-06-30", "2027-01-01", "Rack A"]);
await send("/inventory/bulk", { action: "change", ids: [hw.NAS], warrantyUntil: null });
assert.equal((await row(hw.NAS)).warrantyUntil, null);
await send("/inventory/bulk", r.body.undo);
assert.equal((await row(hw.NAS)).endOfSupport, null);
assert.equal((await row(hw.Switch)).endOfSupport, null);
});
test("a date that is not a date, or nothing to change, is refused and nothing changes", async () => {
assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS], endOfSupport: "2029-02-30" })).status, 400);
assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS], warrantyUntil: "soon" })).status, 400);
assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS] })).status, 400);
assert.equal((await row(hw.NAS)).location, "Rack A");
});
test("an operator is needed", async () => {
who = "viewer";
assert.equal((await send("/inventory/bulk", { action: "change", ids: [hw.NAS], location: "x" })).status, 403);
who = "operator";
});
});
describe("what is written down", () => {
test("each bulk action is one audit entry saying what and how many, and the change log reads them as sentences", async () => {
const entries = await db.select().from(auditLog);
const actions = new Set(entries.map((e) => `${e.category}/${e.action}`));
for (const a of ["server/bulk_tag", "maintenance/bulk_start", "maintenance/bulk_undo", "server/bulk_runs_on", "alert/bulk_ack", "alert/bulk_unack", "alert/bulk_undo", "inventory/bulk_change"]) assert.ok(actions.has(a), a);
const tag = entries.find((e) => e.action === "bulk_tag" && JSON.parse(e.detail!).change === "add_tag")!;
assert.equal(JSON.parse(tag.detail!).tag, "Nightly");
assert.equal(JSON.parse(tag.detail!).changed, 3);
const log = await getChangeLog({ days: 1, limit: 500 });
const titles = log.entries.map((e) => e.title);
assert.ok(titles.some((t) => /^Tag added to 3 servers: Nightly$/.test(t)), "tag");
assert.ok(titles.some((t) => /^Maintenance started on several at once/.test(t)), "maintenance");
assert.ok(titles.some((t) => /alerts? acknowledged at once$/.test(t)), "ack");
assert.ok(titles.some((t) => /pieces? of hardware changed at once$/.test(t)), "hardware");
assert.ok(titles.some((t) => /was undone$/.test(t)), "undo");
});
});
+287
View File
@@ -0,0 +1,287 @@
/**
* Flapping and the noise report: when an alert counts as flapping, what is sent and what is held, how it settles, the flag on the Alerts page, and the report of
* which alerts fire most.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-flapping-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, heartbeats, alertIncidents, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { registerRaise, registerRecovery, hasSettled, flappingKeys, FLAP_WINDOW_MS, SETTLE_MS, keyOfAlertId } = await import("../src/services/flapGuard.js");
const { settleFlappingAlerts } = await import("../src/services/flapMonitor.js");
const { notifyHealthIssues, notifyHealthRecovered } = await import("../src/services/notify.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { buildNoiseReport, mostWithin, hintFor } = await import("../src/services/alertNoise.js");
const { alertsRouter } = await import("../src/routes/alerts.js");
const { updateSettings, setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const entry = (extra: Partial<Parameters<typeof hasSettled>[0]> = {}) => ({ raises: [], flapping: false, since: null, lastEvent: 0, message: "m", ...extra });
describe("when an alert counts as flapping", () => {
test("raised the threshold number of times within an hour: the third is the one that is called flapping; after that, raises are held", () => {
let e = registerRaise(undefined, 0, 3, "disk");
assert.equal(e.verdict, "pass");
e = registerRaise(e.entry, 10 * MIN, 3, "disk");
assert.equal(e.verdict, "pass");
e = registerRaise(e.entry, 20 * MIN, 3, "disk");
assert.equal(e.verdict, "started");
assert.deepEqual([e.entry.flapping, e.entry.since, e.entry.raises.length], [true, 20 * MIN, 3]);
e = registerRaise(e.entry, 25 * MIN, 3, "disk");
assert.equal(e.verdict, "held");
assert.equal(e.entry.flapping, true);
});
test("raises more than an hour apart never add up; a threshold of 0 turns it off", () => {
let e = registerRaise(undefined, 0, 3, "x");
e = registerRaise(e.entry, FLAP_WINDOW_MS + MIN, 3, "x");
e = registerRaise(e.entry, 2 * FLAP_WINDOW_MS + 2 * MIN, 3, "x");
assert.equal(e.verdict, "pass");
assert.equal(e.entry.raises.length, 1, "only the latest is within the window");
let off = registerRaise(undefined, 0, 0, "x");
for (let i = 1; i < 10; i++) off = registerRaise(off.entry, i * MIN, 0, "x");
assert.equal(off.verdict, "pass");
});
test("a recovery is held only while the alert is flapping, and it counts as a moment for the quiet to be measured from", () => {
assert.equal(registerRecovery(undefined, 5).hold, false);
assert.equal(registerRecovery(entry({ raises: [1], flapping: false }), 5).hold, false);
const held = registerRecovery(entry({ flapping: true, lastEvent: 1 }), 50);
assert.deepEqual([held.hold, held.entry?.lastEvent], [true, 50]);
});
test("it has settled after an hour with nothing", () => {
const e = entry({ flapping: true, lastEvent: 0 });
assert.equal(hasSettled(e, SETTLE_MS - 1), false);
assert.equal(hasSettled(e, SETTLE_MS), true);
assert.equal(hasSettled(entry({ flapping: false, lastEvent: 0 }), 10 * SETTLE_MS), false);
});
test("an alert's id says which condition it is about", () => {
assert.equal(keyOfAlertId("disk:disk:server:3:/", "disk"), "disk:server:3:/");
assert.equal(keyOfAlertId("automation:heartbeat:7", "automation"), "heartbeat:7");
});
});
describe("what is sent, what is held, and how it settles", () => {
let hook: Server;
const messages: string[] = [];
const titles = () => messages.map((m) => m.split("\n")[0]);
before(async () => {
hook = createHttpServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c) => chunks.push(c));
req.on("end", () => {
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
res.end("{}");
});
});
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
});
after(() => hook.close());
const issue = { key: "heartbeat:1", message: "Nightly backup hasn't reported", source: "heartbeat", severity: "warning" };
test("raised and cleared again and again: two pairs go out as they are, the third raise is one 'flapping' message, and then it is quiet", async () => {
messages.length = 0;
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
assert.equal(messages.length, 4, "the first two times are as they always were");
await notifyHealthIssues([issue]);
assert.equal(messages.length, 5);
assert.match(messages[4], /^\*\*Homelab Manager — Flapping alert/, "titles go out in bold");
assert.match(messages[4], /↻ Nightly backup hasn't reported — raised 3 times within an hour/);
assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]);
await notifyHealthRecovered([issue]);
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
assert.equal(messages.length, 5, "nothing more while it flaps");
const audit = await db.select().from(auditLog).where(eq(auditLog.action, "flapping_started"));
assert.equal(audit.length, 1);
assert.equal(JSON.parse(audit[0].detail!).key, "heartbeat:1");
});
test("another alert is not affected by it, and one without a key is never held", async () => {
messages.length = 0;
await notifyHealthIssues([{ key: "disk:server:3:/", message: "nas: / is 96% full", severity: "warning" }, { message: "something with no key" }]);
assert.equal(messages.length, 1);
assert.match(messages[0], /nas: \/ is 96% full/);
assert.match(messages[0], /something with no key/);
});
test("the Alerts page marks it as flapping while it is, and not after", async () => {
const [h] = await db.insert(heartbeats).values({ name: "Nightly backup", token: "hb_x".padEnd(24, "x"), periodMinutes: 60, graceMinutes: 5, createdAt: "2026-01-01 00:00:00" }).returning();
// the real heartbeat's key is heartbeat:<id>: make the held alert that one
await setInternalFlag("flapState", JSON.stringify({ [`heartbeat:${h.id}`]: { raises: [Date.now()], flapping: true, since: Date.now(), lastEvent: Date.now(), message: "m" } }));
invalidateAlertsCache();
const alert = (await getAlerts(true)).alerts.find((a) => a.id === `automation:heartbeat:${h.id}`)!;
assert.ok(alert, "the heartbeat is late, so it is an alert");
assert.equal(alert.flapping, true);
await setInternalFlag("flapState", "{}");
invalidateAlertsCache();
assert.equal((await getAlerts(true)).alerts.find((a) => a.id === `automation:heartbeat:${h.id}`)!.flapping, undefined);
await db.delete(heartbeats);
invalidateAlertsCache(); // the heartbeat that was an alert a moment ago is gone
});
test("an hour of quiet settles it: one message says what state it is in; after that it is announced as usual", async () => {
await setInternalFlag("flapState", "{}");
messages.length = 0;
for (let i = 0; i < 3; i++) {
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
}
assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]);
const sent = messages.length;
assert.equal(await settleFlappingAlerts(new Date(Date.now() + 30 * MIN)), 0, "half an hour is not enough");
assert.equal(await settleFlappingAlerts(new Date(Date.now() + SETTLE_MS + MIN)), 1);
assert.equal(messages.length, sent + 1);
assert.match(messages[messages.length - 1], /Alert settled[\s\S]*✓ cleared — Nightly backup hasn't reported/, "nothing says it is still raised, so it says it cleared");
assert.deepEqual([...(await flappingKeys())], []);
assert.equal((await db.select().from(auditLog).where(eq(auditLog.action, "flapping_ended"))).length, 1);
const before = messages.length;
await notifyHealthIssues([issue]);
assert.equal(messages.length, before + 1);
assert.match(messages[messages.length - 1], /Health Alert/);
});
test("turned off, every raise and every recovery is sent", async () => {
await setInternalFlag("flapState", "{}");
await updateSettings({ healthChecks: { flapRaises: 0 } });
messages.length = 0;
for (let i = 0; i < 5; i++) {
await notifyHealthIssues([issue]);
await notifyHealthRecovered([issue]);
}
assert.equal(messages.length, 10);
assert.equal(titles().some((t) => t.includes("Flapping")), false);
await updateSettings({ healthChecks: { flapRaises: 3 } });
});
test("with the notification switched off, it is still tracked, so switching it on does not start from nothing", async () => {
await setInternalFlag("flapState", "{}");
await updateSettings({ notifications: { healthAlerts: false } });
messages.length = 0;
for (let i = 0; i < 3; i++) await notifyHealthIssues([issue]);
assert.equal(messages.length, 0);
assert.deepEqual([...(await flappingKeys())], ["heartbeat:1"]);
await updateSettings({ notifications: { healthAlerts: true } });
await setInternalFlag("flapState", "{}");
});
});
describe("the noise report", () => {
let app: Server;
let base = "";
const incident = (alertKey: string, extra: Partial<typeof alertIncidents.$inferInsert> = {}) => ({ alertKey, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: alertKey, openedAt: "2026-10-10T00:00:00.000Z", lastSeenAt: "2026-10-10T00:00:00.000Z", ...extra });
const at = (day: number, hour: number, min = 0) => new Date(Date.UTC(2026, 9, day, hour, min)).toISOString();
const NOW = new Date("2026-10-19T12:00:00Z");
before(async () => {
await db.insert(users).values({ oidcSub: "viewer", name: "Vic", role: "viewer" });
await db.delete(alertIncidents);
await db.insert(alertIncidents).values([
// a disk that came and went five times, three of them within forty minutes
...[[10, 1, 0, 5], [10, 1, 15, 20], [10, 1, 35, 40], [12, 8, 0, 30], [15, 8, 0, 10]].map(([d, h, m, len]) => incident("disk:disk:server:3:/", { message: "nas: / is 96% full", targetName: "nas", openedAt: at(d, h, m), resolvedAt: new Date(Date.parse(at(d, h, m)) + len * MIN).toISOString(), link: "/servers/3" })),
incident("expiry:secret:1", { category: "expiry", message: "Secret expires", openedAt: at(11, 8) }),
incident("offline:offline:server:2", { category: "offline", message: "elsa stopped reporting", openedAt: at(5, 8), resolvedAt: at(5, 9) }),
incident("offline:offline:server:2", { category: "offline", message: "elsa stopped reporting", openedAt: at(18, 8) }),
// before the period
incident("disk:disk:server:3:/", { message: "old", openedAt: "2026-05-01T00:00:00.000Z", resolvedAt: "2026-05-01T01:00:00.000Z" }),
]);
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/alerts", alertsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/alerts`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the most times within any hour, however the times are spread", () => {
const h = 3600_000;
assert.equal(mostWithin([], h), 0);
assert.equal(mostWithin([0, 10 * MIN, 20 * MIN, 5 * h], h), 3);
assert.equal(mostWithin([0, h, 2 * h], h), 1, "exactly an hour apart is not within an hour");
assert.equal(mostWithin([5 * h, 0, 1 * MIN], h), 2, "in any order");
});
test("what fired most, how long it lasted, which flapped; one that fired once is not noise; an incident before the period is not counted", async () => {
const r = await buildNoiseReport(30, NOW);
assert.deepEqual([r.days, r.incidents, r.distinctAlerts, r.repeating, r.flapped], [30, 8, 3, 2, 1]);
const disk = r.rows[0];
assert.deepEqual([disk.alertId, disk.fired, disk.totalMinutes, disk.averageMinutes, disk.longestMinutes, disk.mostInAnHour, disk.flapped, disk.openNow], ["disk:disk:server:3:/", 5, 105, 21, 40, 3, true, false]);
assert.equal(disk.message, "nas: / is 96% full", "what it said last");
assert.equal(disk.where, "nas");
assert.match(disk.hint, /Raise the limit for this disk/);
const elsa = r.rows[1];
assert.deepEqual([elsa.fired, elsa.openNow, elsa.flapped], [2, true, false]);
assert.match(elsa.hint, /Server offline after/);
assert.equal(elsa.longestMinutes > 60, true, "the open one has been going since it began");
assert.equal(r.rows.some((x) => x.alertId === "expiry:secret:1"), false);
assert.deepEqual(r.byCategory.map((c) => [c.category, c.incidents, c.repeating]), [["disk", 5, 1], ["offline", 2, 1], ["expiry", 1, 0]]);
assert.equal((await buildNoiseReport(180, NOW)).incidents, 9, "a longer period reaches the old one");
assert.equal((await buildNoiseReport(7, NOW)).incidents, 2, "a shorter one leaves most out");
});
test("an alert that is flapping now says so", async () => {
await setInternalFlag("flapState", JSON.stringify({ "disk:server:3:/": { raises: [Date.now()], flapping: true, since: Date.now(), lastEvent: Date.now(), message: "m" } }));
const r = await buildNoiseReport(30, NOW);
assert.equal(r.rows[0].flappingNow, true);
assert.equal(r.rows[1].flappingNow, false);
await setInternalFlag("flapState", "{}");
});
test("advice for what the alert is about", () => {
assert.match(hintFor("monitoring", "monitoring:new-port:server:1:tcp:8080"), /Ports card/);
assert.match(hintFor("automation", "automation:heartbeat:1"), /grace/);
assert.match(hintFor("monitoring", "monitoring:endpoint:2"), /failures in a row/);
assert.match(hintFor("services", "services:failed-units:server:1"), /Services to ignore/);
assert.match(hintFor("tickets", "tickets:x"), /Acknowledge it/);
});
test("everyone signed in can read it; the period is one of four, or thirty", async () => {
const body = await (await fetch(`${base}/noise?days=90`)).json();
assert.equal(body.days, 90);
assert.equal((await (await fetch(`${base}/noise?days=banana`)).json()).days, 30);
assert.equal((await (await fetch(`${base}/noise?days=45`)).json()).days, 30, "not an arbitrary number");
assert.ok(Array.isArray(body.rows));
});
});
+2
View File
@@ -174,6 +174,8 @@ describe("through the channels", () => {
gotify: { enabled: true, url: base.gotify, token: "g" },
ntfy: { enabled: true, url: base.ntfy, topic: "lab" },
webhook: { enabled: true, url: base.webhook },
// These tests raise the same problem again and again as fixtures; they are about routing, not about flapping.
healthChecks: { flapRaises: 0 },
});
proddy = (await db.insert(servers).values({ name: "prod-db", apiTokenHash: "p", apiTokenPrefix: "p", tags: JSON.stringify(["prod"]) }).returning({ id: servers.id }))[0].id;
});