Close an alert: dealt with or accepted, hidden until it gets worse

Close... on the Alerts page hides an alert with no end, with an optional reason
that is also written on its incident. It opens again only if it gets more
serious than when it was closed (said once) or the problem has been gone for
half an hour; Open again brings it back at once.

Built on the snooze table (kind, severity and note columns), so everything that
holds back snoozed alerts holds back closed ones. Audit log, change log and
/api/v1 alerts carry it.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-25 20:55:39 +01:00
1 parent 4ff0b44943
commit a94899865e
24 files changed
+7982 -24

No files matched your search

+4
View File
@@ -1622,6 +1622,10 @@ export const alertSnoozes = sqliteTable(
snoozedBy: text("snoozed_by").notNull(),
snoozedAt: text("snoozed_at").notNull(), // ISO
lastSeenAt: text("last_seen_at").notNull(), // ISO; the last time the alert was there
// A closed alert is a snooze with no end: hidden until it gets worse (a higher severity than `severity`, which is what it was when it was closed) or has gone for half an hour.
kind: text("kind").$type<"snooze" | "closed">().notNull().default("snooze"),
severity: text("severity"), // for a closed one: critical, warning or info as it was
note: text("note"), // for a closed one: why
},
(t) => [uniqueIndex("alert_snoozes_alert_idx").on(t.alertId)],
);
+1 -1
View File
@@ -64,7 +64,7 @@ alertActionsRouter.get("/:token", asyncHandler(async (req, res) => {
continue;
}
const ack = await loadAck(alert.id, now);
const state = alert.snoozed ? `Snoozed until ${esc(new Date(alert.snoozed.until).toISOString().slice(0, 16).replace("T", " "))} UTC.` : ack?.active ? `Acknowledged by ${esc(ack.ackedBy)}.` : alert.silenced ? "Held back (under maintenance, or folded under something that is down)." : "Nobody has acknowledged it.";
const state = alert.closed ? `Closed by ${esc(alert.closed.by)}.` : alert.snoozed ?`Snoozed until ${esc(new Date(alert.snoozed.until).toISOString().slice(0, 16).replace("T", " "))} UTC.` : ack?.active ? `Acknowledged by ${esc(ack.ackedBy)}.` : alert.silenced ? "Held back (under maintenance, or folded under something that is down)." : "Nobody has acknowledged it.";
const form = (action: ActionId, label: string, secondary = false) =>
`<form method="post" action="/api/alert-actions/${esc(req.params.token)}/${action}"><input type="hidden" name="key" value="${esc(key)}"><button type="submit"${secondary ? ' class="secondary"' : ""}>${esc(label)}</button></form>`;
cards += `<div class="card"><div class="sev ${esc(alert.severity)}">${esc(alert.severity)}</div><div class="msg">${esc(alert.message)}</div><div class="state">${state}</div>${form("ack", "Acknowledge")}${alert.target ? form("silence", `Silence for ${SILENCE_MINUTES === 60 ? "an hour" : `${SILENCE_MINUTES} min`}`, true) : ""}${form("snooze", `Snooze until ${untilText}`, true)}<div class="small"><a href="${esc(base + (alert.link ?? "/alerts"))}">Open in Homelab Manager</a></div></div>`;
+45 -2
View File
@@ -2,11 +2,11 @@ import { Router } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { getAlerts, invalidateAlertsCache } from "../services/alerts.js";
import { clearSnooze, setSnooze, SnoozeError } from "../services/alertSnoozes.js";
import { clearSnooze, MAX_CLOSE_NOTE, setClosed, setSnooze, SnoozeError } from "../services/alertSnoozes.js";
import { keyOfAlertId } from "../services/flapGuard.js";
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
import { recordAudit } from "../services/audit.js";
import { openIncidentIds } from "../services/incidents.js";
import { addNote, openIncidentIds } from "../services/incidents.js";
import { buildNoiseReport, NOISE_DAYS } from "../services/alertNoise.js";
import { ticketsForAlerts, type AlertTicketInfo } from "../services/alertTickets.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -133,3 +133,46 @@ alertsRouter.delete("/snooze", requireRole("operator"), asyncHandler(async (req,
invalidateAlertsCache();
res.status(204).end();
}));
const closeSchema = z.object({
alertId: z.string().min(1).max(600),
/** Why — shown on the alert and written into its incident. */
note: z.string().max(MAX_CLOSE_NOTE).nullable().optional(),
});
// "I have dealt with it, or I accept it": the alert is hidden with no end, until it gets worse or has gone for half an hour. The reason goes on the incident, so the history says why.
alertsRouter.put("/close", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = closeSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Say which alert, and keep the note under " + MAX_CLOSE_NOTE + " characters." });
const alert = (await getAlerts(false)).alerts.find((a) => a.id === parsed.data.alertId);
if (!alert) return res.status(404).json({ error: "not_found", message: "That alert isn't there any more." });
const actor = req.currentUser!;
const who = actor.name ?? actor.email ?? actor.oidcSub;
try {
const closed = await setClosed({ alertId: alert.id, conditionKey: keyOfAlertId(alert.id, alert.category), message: alert.message, severity: alert.severity, note: parsed.data.note ?? null, actor: who });
// The incident it is part of says it was closed, by whom and why.
const incidentId = (await openIncidentIds([alert.id])).get(alert.id);
if (incidentId) await addNote(incidentId, who, closed.note ? "Closed: " + closed.note : "Closed without a note.");
await recordAudit({ actor, category: "alert", action: "close", ...auditTargetOf(alert), detail: { alertId: alert.id, message: alert.message, note: closed.note } });
invalidateAlertsCache();
res.json({ closed: { by: closed.snoozedBy, at: closed.snoozedAt, note: closed.note } });
} catch (err) {
if (err instanceof SnoozeError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
// Opens it again: it is shown, counted and announced like any other (and, if it is still there, is a new message to the people who get them).
alertsRouter.delete("/close", requireRole("operator"), asyncHandler(async (req, res) => {
const alertId = typeof req.query.alertId === "string" ? req.query.alertId : "";
if (!alertId) return res.status(400).json({ error: "invalid_id" });
const removed = await clearSnooze(alertId);
if (!removed) return res.status(404).json({ error: "not_found" });
const actor = req.currentUser!;
const alert = (await getAlerts(false)).alerts.find((a) => a.id === alertId);
const incidentId = alert ? (await openIncidentIds([alertId])).get(alertId) : undefined;
if (incidentId) await addNote(incidentId, actor.name ?? actor.email ?? actor.oidcSub, "Opened again.");
await recordAudit({ actor, category: "alert", action: "reopen", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: removed.message } });
invalidateAlertsCache();
res.status(204).end();
}));
+1 -1
View File
@@ -103,7 +103,7 @@ const previousAck = z
})
.nullable();
const snoozeRecord = z.object({ conditionKey: z.string().max(700), message: z.string().max(500), until: z.string().max(40), snoozedBy: z.string().max(200), snoozedAt: z.string().max(40) });
const snoozeRecord = z.object({ conditionKey: z.string().max(700), message: z.string().max(500), until: z.string().max(40), snoozedBy: z.string().max(200), snoozedAt: z.string().max(40), kind: z.enum(["snooze", "closed"]).optional(), severity: z.string().max(20).nullable().optional(), note: z.string().max(500).nullable().optional() });
const alertsBody = z.discriminatedUnion("action", [
/** When they should come back, as a date and time with its zone. */
+39 -9
View File
@@ -30,9 +30,17 @@ export interface SnoozeInfo {
until: string;
snoozedBy: string;
snoozedAt: string;
/** Closed rather than snoozed: it has no end, only the conditions in this file's header. */
closed: boolean;
note: string | null;
}
const shape = (r: typeof alertSnoozes.$inferSelect): SnoozeInfo => ({ until: r.until, snoozedBy: r.snoozedBy, snoozedAt: r.snoozedAt });
const shape = (r: typeof alertSnoozes.$inferSelect): SnoozeInfo => ({ until: r.until, snoozedBy: r.snoozedBy, snoozedAt: r.snoozedAt, closed: r.kind === "closed", note: r.note });
/** A closed alert has no end; this is a date so far off that it never comes, which keeps every place that asks "is it still snoozed?" working as it is. */
export const CLOSED_UNTIL = "9999-12-31T00:00:00.000Z";
const SEVERITY_RANK: Record<string, number> = { info: 0, warning: 1, critical: 2 };
export const MAX_CLOSE_NOTE = 500;
/** The snoozes still running, by alert id. */
export async function loadSnoozes(now: Date = new Date()): Promise<Map<string, SnoozeInfo>> {
@@ -55,9 +63,21 @@ export async function withoutSnoozed<T extends { key?: string }>(items: T[], now
export async function setSnooze(input: { alertId: string; conditionKey: string; message: string; until: Date; actor: string }, now: Date = new Date()): Promise<SnoozeInfo> {
if (!(input.until.getTime() >= now.getTime() + MIN_AHEAD_MS)) throw new SnoozeError(400, "invalid_until", "Choose a time that has not come yet.");
if (input.until.getTime() > now.getTime() + MAX_DAYS * 86_400_000) throw new SnoozeError(400, "invalid_until", `A snooze can be at most ${MAX_DAYS} days.`);
const values = { conditionKey: input.conditionKey, message: input.message.slice(0, 500), until: input.until.toISOString(), snoozedBy: input.actor, snoozedAt: now.toISOString(), lastSeenAt: now.toISOString() };
const values = { conditionKey: input.conditionKey, message: input.message.slice(0, 500), until: input.until.toISOString(), snoozedBy: input.actor, snoozedAt: now.toISOString(), lastSeenAt: now.toISOString(), kind: "snooze" as const, severity: null, note: null };
await db.insert(alertSnoozes).values({ alertId: input.alertId, ...values }).onConflictDoUpdate({ target: alertSnoozes.alertId, set: values });
return { until: values.until, snoozedBy: values.snoozedBy, snoozedAt: values.snoozedAt };
return { until: values.until, snoozedBy: values.snoozedBy, snoozedAt: values.snoozedAt, closed: false, note: null };
}
/**
* Closes an alert: "I have dealt with it, or I accept it". It is hidden like a snoozed one, with no end — until it **gets worse** (a more serious severity than it had when it was
* closed) or has been **gone for half an hour** (the same problem coming back later is new). A number in its message changing (a disk at 91 %, then 92 %) is not a change. `note` says why.
*/
export async function setClosed(input: { alertId: string; conditionKey: string; message: string; severity: string; note: string | null; actor: string }, now: Date = new Date()): Promise<SnoozeInfo> {
const note = (input.note ?? "").trim();
if (note.length > MAX_CLOSE_NOTE) throw new SnoozeError(400, "note_too_long", `Keep the note under ${MAX_CLOSE_NOTE} characters.`);
const values = { conditionKey: input.conditionKey, message: input.message.slice(0, 500), until: CLOSED_UNTIL, snoozedBy: input.actor, snoozedAt: now.toISOString(), lastSeenAt: now.toISOString(), kind: "closed" as const, severity: input.severity, note: note === "" ? null : note };
await db.insert(alertSnoozes).values({ alertId: input.alertId, ...values }).onConflictDoUpdate({ target: alertSnoozes.alertId, set: values });
return { until: values.until, snoozedBy: values.snoozedBy, snoozedAt: values.snoozedAt, closed: true, note: values.note };
}
/** A snooze as it is stored, whole — what an undo puts back. */
@@ -67,12 +87,16 @@ export interface SnoozeRecord {
until: string;
snoozedBy: string;
snoozedAt: string;
/** Left out of an older record: a snooze. */
kind?: "snooze" | "closed";
severity?: string | null;
note?: string | null;
}
/** The snooze on an alert as it stands, expired or not; null when it has none. */
/** The snooze (or the closing) on an alert as it stands, expired or not; null when it has none. */
export async function readSnooze(alertId: string): Promise<SnoozeRecord | null> {
const [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, alertId)).limit(1);
return row ? { conditionKey: row.conditionKey, message: row.message, until: row.until, snoozedBy: row.snoozedBy, snoozedAt: row.snoozedAt } : null;
return row ? { conditionKey: row.conditionKey, message: row.message, until: row.until, snoozedBy: row.snoozedBy, snoozedAt: row.snoozedAt, kind: row.kind, severity: row.severity, note: row.note } : null;
}
/** Puts a snooze back as it was (an undo). One whose time has already passed is not worth restoring: it would only announce the alert again at once, so it is dropped. */
@@ -81,7 +105,7 @@ export async function restoreSnooze(alertId: string, record: SnoozeRecord, now:
await clearSnooze(alertId);
return;
}
const values = { ...record, lastSeenAt: now.toISOString() };
const values = { ...record, kind: record.kind ?? ("snooze" as const), severity: record.severity ?? null, note: record.note ?? null, lastSeenAt: now.toISOString() };
await db.insert(alertSnoozes).values({ alertId, ...values }).onConflictDoUpdate({ target: alertSnoozes.alertId, set: values });
}
@@ -96,11 +120,14 @@ export interface Reminder {
key: string;
message: string;
severity?: string;
/** "ended": a snooze's time is up. "worse": a closed alert got more serious. */
reason?: "ended" | "worse";
}
/**
* One look, with the alerts that are there now: snoozes that have run out are removed, and each whose alert is still there becomes a reminder; the others' alerts are
* marked as seen, and those that have been gone for half an hour are dropped. Returns the reminders to send.
* marked as seen, and those that have been gone for half an hour are dropped. A closed alert that is there with a higher severity than when it was closed is opened again,
* and announced as having got worse. Returns the reminders to send.
*/
export async function settleSnoozes(present: { id: string; message: string; severity: string; key: string }[], now: Date = new Date()): Promise<Reminder[]> {
const rows = await db.select().from(alertSnoozes);
@@ -112,8 +139,11 @@ export async function settleSnoozes(present: { id: string; message: string; seve
const cutoff = new Date(now.getTime() - GONE_AFTER_MS).toISOString();
for (const r of rows) {
const there = byId.get(r.alertId);
if (Date.parse(r.until) <= now.getTime()) {
if (there) reminders.push({ key: r.conditionKey, message: there.message, severity: there.severity });
if (r.kind === "closed" && there && (SEVERITY_RANK[there.severity] ?? 0) > (SEVERITY_RANK[r.severity ?? ""] ?? 0)) {
reminders.push({ key: r.conditionKey, message: there.message, severity: there.severity, reason: "worse" });
done.push(r.id);
} else if (Date.parse(r.until) <= now.getTime()) {
if (there) reminders.push({ key: r.conditionKey, message: there.message, severity: there.severity, reason: "ended" });
done.push(r.id);
} else if (there) seen.push(r.id);
else if (r.lastSeenAt < cutoff) done.push(r.id);
+2
View File
@@ -25,6 +25,8 @@ export interface Alert {
affects?: string[];
/** Snoozed until then (services/alertSnoozes.ts): held back like one under maintenance, and announced again if it is still there. */
snoozed?: { until: string; by: string };
/** Closed by somebody (services/alertSnoozes.ts): held back like a snoozed one, with no end — until it gets worse, or has gone for half an hour. */
closed?: { by: string; at: string; note: string | null };
/** Raised and cleared over and over: its raised and cleared messages are being held back (services/flapGuard.ts). */
flapping?: boolean;
/** What a maintenance window would have to be on to silence it, when there is such a thing — which is what "Silence" acts on. */
+3 -2
View File
@@ -684,12 +684,13 @@ export async function collectAlerts(): Promise<AlertsReport> {
if (flapping.size > 0) for (const a of alerts) if (flapping.has(keyOfAlertId(a.id, a.category))) a.flapping = true;
// Those somebody has snoozed are held back like those under maintenance, with when they come back.
const snoozes = await loadSnoozes().catch(() => new Map<string, { until: string; snoozedBy: string }>());
const snoozes = await loadSnoozes().catch(() => new Map<string, { until: string; snoozedBy: string; snoozedAt: string; closed: boolean; note: string | null }>());
if (snoozes.size > 0) {
for (const a of alerts) {
const sn = snoozes.get(a.id);
if (sn) {
a.snoozed = { until: sn.until, by: sn.snoozedBy };
if (sn.closed) a.closed = { by: sn.snoozedBy, at: sn.snoozedAt, note: sn.note };
else a.snoozed = { until: sn.until, by: sn.snoozedBy };
a.silenced = true;
}
}
+4
View File
@@ -223,6 +223,10 @@ export function describeChange(category: string, action: string, detail: Record<
return { title: `Alert rule removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "alert/snooze":
return { title: `Alert snoozed${typeof detail.message === "string" ? `: ${detail.message}` : ""}`, tone: "neutral", detail: str(detail.until) ? `until ${str(detail.until)}` : null };
case "alert/close":
return { title: `Alert closed${typeof detail.message === "string" ? `: ${detail.message}` : ""}`, tone: "neutral", detail: str(detail.note) };
case "alert/reopen":
return { title: `Closed alert opened again${typeof detail.message === "string" ? `: ${detail.message}` : ""}`, tone: "neutral", detail: null };
case "alert/unsnooze":
return { title: `Snooze ended early${typeof detail.message === "string" ? `: ${detail.message}` : ""}`, tone: "neutral", detail: null };
case "alert/flapping_started":
+9
View File
@@ -517,6 +517,15 @@ export async function notifySnoozeEnded(items: { message: string; key?: string;
}), (i) => i.key);
}
/** Alerts that somebody closed and that have since got more serious: opened again, and said once. */
export async function notifyClosedWorse(items: { message: string; key?: string; severity?: string }[]): Promise<void> {
if (items.length === 0) return;
await notifyGrouped(items, issueMeta, (group) => ({
title: "Homelab Manager — Closed alert got worse",
message: `${group.length === 1 ? "An alert you closed is" : `${group.length} alerts you closed are`} more serious now, so ${group.length === 1 ? "it is" : "they are"} open again:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`,
}), (i) => i.key);
}
/** An alert that has just been called flapping: said once, with how often it has come, instead of each time. */
export async function notifyFlapping(items: { message: string; key?: string; severity?: string; count: number }[]): Promise<void> {
if (items.length === 0) return;
+2
View File
@@ -100,6 +100,8 @@ export async function alertsV1() {
flapping: !!a.flapping,
// Snoozed alerts are also `silenced`; this says until when they are.
snoozedUntil: a.snoozed?.until ?? null,
// Closed ones are `silenced` too; this says by whom and why. They have no end.
closed: a.closed ? { by: a.closed.by, at: a.closed.at, note: a.closed.note } : null,
target: a.target,
causedBy: a.causedBy ?? null,
affects: a.affects ?? [],
+3 -2
View File
@@ -4,7 +4,7 @@
import { getAlerts, invalidateAlertsCache } from "./alerts.js";
import { keyOfAlertId } from "./flapGuard.js";
import { settleSnoozes } from "./alertSnoozes.js";
import { notifySnoozeEnded } from "./notify.js";
import { notifyClosedWorse, notifySnoozeEnded } from "./notify.js";
/** One look: returns how many reminders were sent. */
export async function remindAboutSnoozed(now: Date = new Date()): Promise<number> {
@@ -13,7 +13,8 @@ export async function remindAboutSnoozed(now: Date = new Date()): Promise<number
if (reminders.length > 0) {
// The page should show them again at once, not when its stored result expires.
invalidateAlertsCache();
await notifySnoozeEnded(reminders);
await notifySnoozeEnded(reminders.filter((r) => r.reason !== "worse"));
await notifyClosedWorse(reminders.filter((r) => r.reason === "worse"));
}
return reminders.length;
}