Flapping alerts, a noise report and bulk actions with undo

Flapping: an alert raised three times within an hour is announced once as
flapping, its further raise/clear messages are held until it has been calm
for an hour, and one message then says whether it is still active. Event
webhooks still hear everything. The state survives restarts.

Alerts > Noise lists the alerts that keep coming back (7-180 days): how often,
how long, most in an hour, and a hint on what helps.

Bulk actions on Alerts (acknowledge, remove acknowledgement, silence), Servers
(tag, silence, runs on) and Inventory (location and dates, print labels). Each
reports what it left alone and why, and has an Undo on the toast.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 23:55:09 +02:00
1 parent be6c5552bd
commit 73a06ebe63
35 files changed
+2546 -17

No files matched your search

+290
View File
@@ -0,0 +1,290 @@
/**
* Bulk actions: do the same thing to several alerts, servers or hardware entries at once. Each action says what it changed and returns, with it, the body that
* puts everything back — so "Undo" is only sending that body to the same address again, and an undo needs nothing remembered on the server. Whatever could not be
* done (an alert that has gone, a server already at its limit of tags, a change that would make a loop) is listed with the reason and does not stop the rest.
*/
import { and, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, maintenanceWindows, servers } from "../db/schema.js";
import { invalidateAlertsCache, getAlerts } from "./alerts.js";
import { clearAck, restoreAck, setAck, type AckInfo, type SetAckInput } from "./alertAcks.js";
import { impactOf } from "./dependencies.js";
import { isRealDate } from "./inventory.js";
import { listActiveWindows, startOrExtendWindow } from "./maintenance.js";
import { loadParentRows, wouldLoop } from "./serverParents.js";
import { InvalidTagError, normalizeTags, parseTags } from "./serverTags.js";
export const MAX_BULK = 200;
export interface Skipped {
id: string | number;
name: string;
reason: string;
}
export interface BulkResult<U> {
changed: number;
skipped: Skipped[];
/** The body that puts it back, or null when nothing changed. */
undo: U | null;
}
const unique = <T>(list: T[]): T[] => [...new Set(list)];
// ─── Maintenance (silence), shared by servers and alerts ───────────────────
export interface SilenceUndo {
action: "undo_silence";
/** Windows these made: ended again. */
created: number[];
/** Windows that were already there and were extended: put back as they were. */
restored: { id: number; endsAt: string; reason: string | null }[];
}
type Target = { type: "server" | "integration" | "dns_provider"; id: number };
/** Puts each target under maintenance (and, if asked, what depends on a server). Targets that are already under it have their window extended. */
export async function silenceTargets(targets: Target[], minutes: number, reason: string | null, createdBy: string, withDependents: boolean, now: Date = new Date()): Promise<{ silenced: { type: string; id: number; name: string }[]; skipped: Skipped[]; undo: SilenceUndo | null }> {
const wanted = new Map<string, Target & { why: string | null }>();
for (const t of targets) wanted.set(`${t.type}:${t.id}`, { ...t, why: reason });
if (withDependents) {
for (const t of targets) {
if (t.type !== "server" && t.type !== "integration") continue;
const impact = await impactOf(t.type, t.id, now);
for (const d of impact?.dependents ?? []) {
const key = `${d.type}:${d.id}`;
if (!wanted.has(key)) wanted.set(key, { type: d.type, id: d.id, why: `Depends on ${impact?.target.name ?? "a server in the same change"}${reason ? ` — ${reason}` : ""}`.slice(0, 200) });
}
}
}
const active = await listActiveWindows(now);
const created: number[] = [];
const restored: SilenceUndo["restored"] = [];
const silenced: { type: string; id: number; name: string }[] = [];
const skipped: Skipped[] = [];
for (const t of wanted.values()) {
const existing = active.find((w) => w.targetType === t.type && w.targetId === t.id);
const started = await startOrExtendWindow(t.type, t.id, minutes, t.why, createdBy, now);
if (!started) {
skipped.push({ id: t.id, name: `${t.type} #${t.id}`, reason: "it no longer exists" });
continue;
}
silenced.push({ type: t.type, id: t.id, name: started.target.name });
if (started.extended && existing) restored.push({ id: existing.id, endsAt: existing.endsAt, reason: existing.reason });
else created.push(started.window.id);
}
invalidateAlertsCache();
return { silenced, skipped, undo: silenced.length > 0 ? { action: "undo_silence", created, restored } : null };
}
export async function undoSilence(undo: Pick<SilenceUndo, "created" | "restored">, now: Date = new Date()): Promise<number> {
if (undo.created.length > 0) await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(inArray(maintenanceWindows.id, undo.created));
for (const r of undo.restored) await db.update(maintenanceWindows).set({ endsAt: r.endsAt, reason: r.reason }).where(eq(maintenanceWindows.id, r.id));
invalidateAlertsCache();
return undo.created.length + undo.restored.length;
}
// ─── Servers ───────────────────────────────────────────────────────────────
export interface TagsUndo {
action: "set_tags";
items: { id: number; tags: string[] }[];
}
/** Adds or takes away a tag on each server. A server that already has it (or lacks it), or has no room for another, is left as it was. */
export async function changeServerTag(ids: number[], change: { add: string } | { remove: string }): Promise<BulkResult<TagsUndo>> {
const rows = await db.select().from(servers).where(inArray(servers.id, unique(ids)));
const tag = normalizeTags(["add" in change ? change.add : change.remove])[0];
if (!tag) throw new InvalidTagError("A tag needs a name.");
const skipped: Skipped[] = unique(ids).filter((id) => !rows.some((r) => r.id === id)).map((id) => ({ id, name: `server #${id}`, reason: "it no longer exists" }));
const items: TagsUndo["items"] = [];
for (const r of rows) {
const before = parseTags(r.tags);
let after: string[];
if ("add" in change) {
if (before.includes(tag)) {
skipped.push({ id: r.id, name: r.name, reason: "it already has that tag" });
continue;
}
try {
after = normalizeTags([...before, tag]);
} catch (err) {
if (err instanceof InvalidTagError) {
skipped.push({ id: r.id, name: r.name, reason: err.message });
continue;
}
throw err;
}
} else {
if (!before.includes(tag)) {
skipped.push({ id: r.id, name: r.name, reason: "it doesn't have that tag" });
continue;
}
after = before.filter((t) => t !== tag);
}
await db.update(servers).set({ tags: after.length > 0 ? JSON.stringify(after) : null }).where(eq(servers.id, r.id));
items.push({ id: r.id, tags: before });
}
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "set_tags", items } : null };
}
/** Sets each server's tags to exactly what is given (the undo of a tag change). */
export async function setServerTags(items: { id: number; tags: string[] }[]): Promise<number> {
let n = 0;
for (const item of items) {
const tags = normalizeTags(item.tags);
const done = await db.update(servers).set({ tags: tags.length > 0 ? JSON.stringify(tags) : null }).where(eq(servers.id, item.id)).returning({ id: servers.id });
n += done.length;
}
return n;
}
export interface ParentUndo {
action: "restore_parents";
items: { id: number; parentServerId: number | null; noParent: boolean }[];
}
/** Sets which server each runs on. `parentServerId` null goes back to working it out from the Proxmox node; a change that would make a loop is skipped. */
export async function setServerParents(ids: number[], parentServerId: number | null): Promise<BulkResult<ParentUndo>> {
const rows = await loadParentRows();
const skipped: Skipped[] = [];
const items: ParentUndo["items"] = [];
if (parentServerId !== null && !rows.some((r) => r.id === parentServerId)) throw new RangeError("That server doesn't exist.");
for (const id of unique(ids)) {
const row = rows.find((r) => r.id === id);
if (!row) {
skipped.push({ id, name: `server #${id}`, reason: "it no longer exists" });
continue;
}
if (parentServerId !== null) {
if (parentServerId === id) {
skipped.push({ id, name: row.name, reason: "a server can't run on itself" });
continue;
}
if (wouldLoop(rows, id, parentServerId)) {
skipped.push({ id, name: row.name, reason: "that server already runs on this one, so each would be waiting for the other" });
continue;
}
}
items.push({ id, parentServerId: row.parentServerId ?? null, noParent: row.noParent ?? false });
await db.update(servers).set({ parentServerId, noParent: false }).where(eq(servers.id, id));
row.parentServerId = parentServerId;
row.noParent = false;
}
if (items.length > 0) invalidateAlertsCache();
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_parents", items } : null };
}
export async function restoreServerParents(items: ParentUndo["items"]): Promise<number> {
let n = 0;
for (const i of items) {
const done = await db.update(servers).set({ parentServerId: i.parentServerId, noParent: i.noParent }).where(eq(servers.id, i.id)).returning({ id: servers.id });
n += done.length;
}
if (n > 0) invalidateAlertsCache();
return n;
}
// ─── Alerts ────────────────────────────────────────────────────────────────
export interface AcksUndo {
action: "restore_acks";
items: { alertId: string; previous: AckInfo | null }[];
}
/** Acknowledges each alert the same way. Alerts that are no longer there are skipped. */
export async function acknowledgeAlerts(alertIds: string[], input: Omit<SetAckInput, "alertId">): Promise<BulkResult<AcksUndo>> {
const present = new Map((await getAlerts(false)).alerts.map((a) => [a.id, a]));
const skipped: Skipped[] = [];
const items: AcksUndo["items"] = [];
for (const alertId of unique(alertIds)) {
if (!present.has(alertId)) {
skipped.push({ id: alertId, name: alertId, reason: "it isn't there any more" });
continue;
}
const { before } = await setAck({ ...input, alertId });
items.push({ alertId, previous: before });
}
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_acks", items } : null };
}
/** Takes the acknowledgement off each alert that has one. */
export async function unacknowledgeAlerts(alertIds: string[]): Promise<BulkResult<AcksUndo>> {
const skipped: Skipped[] = [];
const items: AcksUndo["items"] = [];
for (const alertId of unique(alertIds)) {
const before = await clearAck(alertId);
if (!before) {
skipped.push({ id: alertId, name: alertId, reason: "nobody had acknowledged it" });
continue;
}
items.push({ alertId, previous: before });
}
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_acks", items } : null };
}
/** Puts each alert's acknowledgement back as it was (or takes it off, if it had none). */
export async function restoreAcks(items: AcksUndo["items"]): Promise<number> {
for (const i of items) {
if (i.previous === null) await clearAck(i.alertId);
else await restoreAck(i.alertId, i.previous);
}
return items.length;
}
/** The maintenance targets of the alerts given (those with none can't be silenced). */
export async function targetsOfAlerts(alertIds: string[]): Promise<{ targets: Target[]; skipped: Skipped[] }> {
const present = new Map((await getAlerts(false)).alerts.map((a) => [a.id, a]));
const targets = new Map<string, Target>();
const skipped: Skipped[] = [];
for (const alertId of unique(alertIds)) {
const alert = present.get(alertId);
if (!alert) skipped.push({ id: alertId, name: alertId, reason: "it isn't there any more" });
else if (!alert.target) skipped.push({ id: alertId, name: alert.message.slice(0, 80), reason: "there is nothing to put under maintenance for it" });
else targets.set(`${alert.target.type}:${alert.target.id}`, { type: alert.target.type, id: alert.target.id });
}
return { targets: [...targets.values()], skipped };
}
// ─── Hardware ──────────────────────────────────────────────────────────────
export interface AssetsUndo {
action: "restore";
items: { id: number; location: string | null; warrantyUntil: string | null; endOfSupport: string | null }[];
}
export class BulkInputError extends Error {}
/** What a blank in a form means: nothing, not an empty string. */
const blank = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
/** Sets the location and/or the dates of each piece of hardware. Whatever is not given is left as it is; a blank clears. */
export async function changeAssets(ids: number[], change: { location?: string | null; warrantyUntil?: string | null; endOfSupport?: string | null }): Promise<BulkResult<AssetsUndo>> {
const set: Partial<typeof assets.$inferInsert> = {};
if (change.location !== undefined) set.location = blank(change.location);
for (const field of ["warrantyUntil", "endOfSupport"] as const) {
if (change[field] === undefined) continue;
const v = blank(change[field]);
if (v !== null && !isRealDate(v)) throw new BulkInputError("Use a real date, like 2027-06-30.");
set[field] = v;
}
if (Object.keys(set).length === 0) throw new BulkInputError("Say what to change.");
const rows = await db.select().from(assets).where(inArray(assets.id, unique(ids)));
const skipped: Skipped[] = unique(ids).filter((id) => !rows.some((r) => r.id === id)).map((id) => ({ id, name: `hardware #${id}`, reason: "it no longer exists" }));
const items: AssetsUndo["items"] = [];
for (const r of rows) {
items.push({ id: r.id, location: r.location, warrantyUntil: r.warrantyUntil, endOfSupport: r.endOfSupport });
await db.update(assets).set({ ...set, updatedAt: new Date().toISOString() }).where(eq(assets.id, r.id));
}
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore", items } : null };
}
export async function restoreAssets(items: AssetsUndo["items"]): Promise<number> {
let n = 0;
for (const i of items) {
const done = await db.update(assets).set({ location: i.location, warrantyUntil: i.warrantyUntil, endOfSupport: i.endOfSupport, updatedAt: new Date().toISOString() }).where(and(eq(assets.id, i.id))).returning({ id: assets.id });
n += done.length;
}
return n;
}