Flapping alerts, a noise report and bulk actions with undo
Flapping: an alert raised three times within an hour is announced once as flapping, its further raise/clear messages are held until it has been calm for an hour, and one message then says whether it is still active. Event webhooks still hear everything. The state survives restarts. Alerts > Noise lists the alerts that keep coming back (7-180 days): how often, how long, most in an hour, and a hint on what helps. Bulk actions on Alerts (acknowledge, remove acknowledgement, silence), Servers (tag, silence, runs on) and Inventory (location and dates, print labels). Each reports what it left alone and why, and has an Undo on the toast. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
be6c5552bd
commit
73a06ebe63
35 files changed
+2546
-17
No files matched your search
@@ -5,6 +5,7 @@ import { getAlerts } from "../services/alerts.js";
|
||||
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { openIncidentIds } from "../services/incidents.js";
|
||||
import { buildNoiseReport, NOISE_DAYS } from "../services/alertNoise.js";
|
||||
import { ticketsForAlerts, type AlertTicketInfo } from "../services/alertTickets.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
@@ -41,6 +42,12 @@ alertsRouter.get("/", asyncHandler(async (req, res) => {
|
||||
});
|
||||
}));
|
||||
|
||||
// Which alerts fire most often and how long they last, from the incident record — for everyone signed in, like the history it comes from. ?days=7|30|90|180.
|
||||
alertsRouter.get("/noise", asyncHandler(async (req, res) => {
|
||||
const asked = Number(req.query.days);
|
||||
res.json(await buildNoiseReport((NOISE_DAYS as readonly number[]).includes(asked) ? asked : 30));
|
||||
}));
|
||||
|
||||
// Who an alert can be handed to.
|
||||
alertsRouter.get("/assignees", requireRole("operator"), asyncHandler(async (_req, res) => {
|
||||
res.json({ users: await assignableUsers() });
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
import { Router } from "express";
|
||||
import { z } from "zod";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { AckError, MAX_HOURS, MAX_NOTE } from "../services/alertAcks.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { acknowledgeAlerts, BulkInputError, changeAssets, changeServerTag, MAX_BULK, restoreAcks, restoreAssets, restoreServerParents, setServerParents, setServerTags, silenceTargets, targetsOfAlerts, undoSilence, unacknowledgeAlerts } from "../services/bulkActions.js";
|
||||
import { InvalidTagError } from "../services/serverTags.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
// Bulk actions are the single-item ones done to several at once, so each needs the same role the single one does; and every one returns the body that puts it back,
|
||||
// which is all an "Undo" sends. One audit entry says what was done to how many.
|
||||
|
||||
const intIds = z.array(z.number().int().positive()).min(1).max(MAX_BULK);
|
||||
const stringIds = z.array(z.string().min(1).max(600)).min(1).max(MAX_BULK);
|
||||
const minutes = z.number().int().min(5).max(7 * 24 * 60);
|
||||
const reason = z.string().trim().max(200).optional();
|
||||
|
||||
const detailIds = (ids: unknown[]) => ids.slice(0, 50);
|
||||
const bad = (res: import("express").Response, message: string) => res.status(400).json({ error: "invalid_body", message });
|
||||
const nameOf = (req: import("express").Request) => {
|
||||
const u = req.currentUser!;
|
||||
return u.name ?? u.email ?? u.oidcSub;
|
||||
};
|
||||
|
||||
// ─── Servers ───────────────────────────────────────────────────────────────
|
||||
|
||||
export const serversBulkRouter = Router();
|
||||
serversBulkRouter.use(requireAuth);
|
||||
|
||||
const serversBody = z.discriminatedUnion("action", [
|
||||
z.object({ action: z.literal("add_tag"), ids: intIds, tag: z.string().min(1).max(100) }),
|
||||
z.object({ action: z.literal("remove_tag"), ids: intIds, tag: z.string().min(1).max(100) }),
|
||||
z.object({ action: z.literal("set_tags"), items: z.array(z.object({ id: z.number().int().positive(), tags: z.array(z.string().max(100)).max(50) })).min(1).max(MAX_BULK) }),
|
||||
z.object({ action: z.literal("silence"), ids: intIds, minutes, reason, withDependents: z.boolean().optional() }),
|
||||
z.object({ action: z.literal("undo_silence"), created: z.array(z.number().int().positive()).max(MAX_BULK * 4), restored: z.array(z.object({ id: z.number().int().positive(), endsAt: z.string().max(40), reason: z.string().max(200).nullable() })).max(MAX_BULK * 4) }),
|
||||
z.object({ action: z.literal("set_parent"), ids: intIds, parentServerId: z.number().int().positive().nullable() }),
|
||||
z.object({ action: z.literal("restore_parents"), items: z.array(z.object({ id: z.number().int().positive(), parentServerId: z.number().int().positive().nullable(), noParent: z.boolean() })).min(1).max(MAX_BULK) }),
|
||||
]);
|
||||
|
||||
serversBulkRouter.post("/", asyncHandler(async (req, res) => {
|
||||
const parsed = serversBody.safeParse(req.body);
|
||||
if (!parsed.success) return bad(res, parsed.error.issues[0]?.message ?? `Choose up to ${MAX_BULK} servers and what to do.`);
|
||||
const body = parsed.data;
|
||||
const actor = req.currentUser!;
|
||||
// Tags and maintenance are an operator's, like one server's; which server another runs on is an administrator's.
|
||||
const needsAdmin = body.action === "set_parent" || body.action === "restore_parents";
|
||||
if (actor.role !== "admin" && (needsAdmin || actor.role !== "operator")) return res.status(403).json({ error: "forbidden", message: needsAdmin ? "Changing which server another runs on needs an administrator." : "That needs an operator." });
|
||||
try {
|
||||
switch (body.action) {
|
||||
case "add_tag":
|
||||
case "remove_tag": {
|
||||
const r = await changeServerTag(body.ids, body.action === "add_tag" ? { add: body.tag } : { remove: body.tag });
|
||||
await recordAudit({ actor, category: "server", action: "bulk_tag", detail: { change: body.action, tag: body.tag, changed: r.changed, ids: detailIds(body.ids) } });
|
||||
return res.json(r);
|
||||
}
|
||||
case "set_tags": {
|
||||
const changed = await setServerTags(body.items);
|
||||
await recordAudit({ actor, category: "server", action: "bulk_tag", detail: { change: "undo", changed, ids: detailIds(body.items.map((i) => i.id)) } });
|
||||
return res.json({ changed, skipped: [], undo: null });
|
||||
}
|
||||
case "silence": {
|
||||
const r = await silenceTargets(body.ids.map((id) => ({ type: "server" as const, id })), body.minutes, body.reason || "Maintenance on several servers", nameOf(req), body.withDependents === true);
|
||||
await recordAudit({ actor, category: "maintenance", action: "bulk_start", detail: { servers: r.silenced.map((s) => s.name).slice(0, 50), minutes: body.minutes, reason: body.reason ?? null, withDependents: body.withDependents === true } });
|
||||
return res.json({ changed: r.silenced.length, skipped: r.skipped, undo: r.undo, silenced: r.silenced });
|
||||
}
|
||||
case "undo_silence": {
|
||||
const changed = await undoSilence(body);
|
||||
await recordAudit({ actor, category: "maintenance", action: "bulk_undo", detail: { windows: changed } });
|
||||
return res.json({ changed, skipped: [], undo: null });
|
||||
}
|
||||
case "set_parent": {
|
||||
const r = await setServerParents(body.ids, body.parentServerId);
|
||||
await recordAudit({ actor, category: "server", action: "bulk_runs_on", detail: { parentServerId: body.parentServerId, changed: r.changed, ids: detailIds(body.ids) } });
|
||||
return res.json(r);
|
||||
}
|
||||
case "restore_parents": {
|
||||
const changed = await restoreServerParents(body.items);
|
||||
await recordAudit({ actor, category: "server", action: "bulk_runs_on", detail: { change: "undo", changed } });
|
||||
return res.json({ changed, skipped: [], undo: null });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof InvalidTagError) return res.status(400).json({ error: "invalid_tag", message: err.message });
|
||||
if (err instanceof RangeError) return res.status(400).json({ error: "invalid_parent", message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
// ─── Alerts ────────────────────────────────────────────────────────────────
|
||||
|
||||
export const alertsBulkRouter = Router();
|
||||
alertsBulkRouter.use(requireAuth, requireRole("operator"));
|
||||
|
||||
const previousAck = z
|
||||
.object({
|
||||
note: z.string().max(MAX_NOTE),
|
||||
assignee: z.object({ id: z.number().int().positive().nullable(), name: z.string().max(200) }).nullable(),
|
||||
ackedBy: z.string().max(200),
|
||||
ackedAt: z.string().max(40),
|
||||
until: z.string().max(40).nullable(),
|
||||
active: z.boolean(),
|
||||
})
|
||||
.nullable();
|
||||
|
||||
const alertsBody = z.discriminatedUnion("action", [
|
||||
z.object({ action: z.literal("acknowledge"), alertIds: stringIds, note: z.string().max(MAX_NOTE).default(""), assigneeId: z.number().int().positive().nullable().default(null), hours: z.number().int().min(1).max(MAX_HOURS).nullable().default(null) }),
|
||||
z.object({ action: z.literal("unacknowledge"), alertIds: stringIds }),
|
||||
z.object({ action: z.literal("restore_acks"), items: z.array(z.object({ alertId: z.string().min(1).max(600), previous: previousAck })).min(1).max(MAX_BULK) }),
|
||||
z.object({ action: z.literal("silence"), alertIds: stringIds, minutes, reason }),
|
||||
z.object({ action: z.literal("undo_silence"), created: z.array(z.number().int().positive()).max(MAX_BULK * 4), restored: z.array(z.object({ id: z.number().int().positive(), endsAt: z.string().max(40), reason: z.string().max(200).nullable() })).max(MAX_BULK * 4) }),
|
||||
]);
|
||||
|
||||
alertsBulkRouter.post("/", asyncHandler(async (req, res) => {
|
||||
const parsed = alertsBody.safeParse(req.body);
|
||||
if (!parsed.success) return bad(res, parsed.error.issues[0]?.message ?? `Choose up to ${MAX_BULK} alerts and what to do.`);
|
||||
const body = parsed.data;
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
switch (body.action) {
|
||||
case "acknowledge": {
|
||||
const r = await acknowledgeAlerts(body.alertIds, { note: body.note, assigneeId: body.assigneeId, hours: body.hours, actor: nameOf(req) });
|
||||
await recordAudit({ actor, category: "alert", action: "bulk_ack", detail: { changed: r.changed, note: body.note, hours: body.hours, alerts: detailIds(body.alertIds) } });
|
||||
return res.json(r);
|
||||
}
|
||||
case "unacknowledge": {
|
||||
const r = await unacknowledgeAlerts(body.alertIds);
|
||||
await recordAudit({ actor, category: "alert", action: "bulk_unack", detail: { changed: r.changed, alerts: detailIds(body.alertIds) } });
|
||||
return res.json(r);
|
||||
}
|
||||
case "restore_acks": {
|
||||
const changed = await restoreAcks(body.items);
|
||||
await recordAudit({ actor, category: "alert", action: "bulk_undo", detail: { changed } });
|
||||
return res.json({ changed, skipped: [], undo: null });
|
||||
}
|
||||
case "silence": {
|
||||
const { targets, skipped } = await targetsOfAlerts(body.alertIds);
|
||||
const r = await silenceTargets(targets, body.minutes, body.reason || "Maintenance, from several alerts", nameOf(req), false);
|
||||
await recordAudit({ actor, category: "maintenance", action: "bulk_start", detail: { targets: r.silenced.map((s) => s.name).slice(0, 50), minutes: body.minutes, reason: body.reason ?? null, fromAlerts: body.alertIds.length } });
|
||||
return res.json({ changed: r.silenced.length, skipped: [...skipped, ...r.skipped], undo: r.undo, silenced: r.silenced });
|
||||
}
|
||||
case "undo_silence": {
|
||||
const changed = await undoSilence(body);
|
||||
await recordAudit({ actor, category: "maintenance", action: "bulk_undo", detail: { windows: changed } });
|
||||
return res.json({ changed, skipped: [], undo: null });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof AckError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
// ─── Hardware ──────────────────────────────────────────────────────────────
|
||||
|
||||
export const inventoryBulkRouter = Router();
|
||||
inventoryBulkRouter.use(requireAuth, requireRole("operator"));
|
||||
|
||||
const dateOrBlank = z.string().max(10).nullable();
|
||||
const inventoryBody = z.discriminatedUnion("action", [
|
||||
z.object({ action: z.literal("change"), ids: intIds, location: z.string().max(120).nullable().optional(), warrantyUntil: dateOrBlank.optional(), endOfSupport: dateOrBlank.optional() }),
|
||||
z.object({ action: z.literal("restore"), items: z.array(z.object({ id: z.number().int().positive(), location: z.string().max(120).nullable(), warrantyUntil: dateOrBlank, endOfSupport: dateOrBlank })).min(1).max(MAX_BULK) }),
|
||||
]);
|
||||
|
||||
inventoryBulkRouter.post("/", asyncHandler(async (req, res) => {
|
||||
const parsed = inventoryBody.safeParse(req.body);
|
||||
if (!parsed.success) return bad(res, parsed.error.issues[0]?.message ?? `Choose up to ${MAX_BULK} pieces of hardware and what to change.`);
|
||||
const body = parsed.data;
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
if (body.action === "restore") {
|
||||
const changed = await restoreAssets(body.items);
|
||||
await recordAudit({ actor, category: "inventory", action: "bulk_change", detail: { change: "undo", changed } });
|
||||
return res.json({ changed, skipped: [], undo: null });
|
||||
}
|
||||
const { ids, action: _a, ...change } = body;
|
||||
const r = await changeAssets(ids, change);
|
||||
await recordAudit({ actor, category: "inventory", action: "bulk_change", detail: { ...change, changed: r.changed, ids: detailIds(ids) } });
|
||||
return res.json(r);
|
||||
} catch (err) {
|
||||
if (err instanceof BulkInputError) return bad(res, err.message);
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
@@ -144,7 +144,7 @@ const updateSchema = z.object({
|
||||
.partial()
|
||||
.optional(),
|
||||
healthChecks: z
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean() })
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20) })
|
||||
.partial()
|
||||
.optional(),
|
||||
quietHours: z
|
||||
|
||||
Reference in new issue
Block a user