Close an alert: dealt with or accepted, hidden until it gets worse
Close... on the Alerts page hides an alert with no end, with an optional reason that is also written on its incident. It opens again only if it gets more serious than when it was closed (said once) or the problem has been gone for half an hour; Open again brings it back at once. Built on the snooze table (kind, severity and note columns), so everything that holds back snoozed alerts holds back closed ones. Audit log, change log and /api/v1 alerts carry it. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4ff0b44943
commit
a94899865e
24 files changed
+7982
-24
No files matched your search
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE `alert_snoozes` ADD `kind` text DEFAULT 'snooze' NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE `alert_snoozes` ADD `severity` text;--> statement-breakpoint
|
||||
ALTER TABLE `alert_snoozes` ADD `note` text;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -449,6 +449,13 @@
|
||||
"when": 1792875717002,
|
||||
"tag": "0063_lucky_talkback",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 64,
|
||||
"version": "6",
|
||||
"when": 1792957211279,
|
||||
"tag": "0064_soft_wildside",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1622,6 +1622,10 @@ export const alertSnoozes = sqliteTable(
|
||||
snoozedBy: text("snoozed_by").notNull(),
|
||||
snoozedAt: text("snoozed_at").notNull(), // ISO
|
||||
lastSeenAt: text("last_seen_at").notNull(), // ISO; the last time the alert was there
|
||||
// A closed alert is a snooze with no end: hidden until it gets worse (a higher severity than `severity`, which is what it was when it was closed) or has gone for half an hour.
|
||||
kind: text("kind").$type<"snooze" | "closed">().notNull().default("snooze"),
|
||||
severity: text("severity"), // for a closed one: critical, warning or info as it was
|
||||
note: text("note"), // for a closed one: why
|
||||
},
|
||||
(t) => [uniqueIndex("alert_snoozes_alert_idx").on(t.alertId)],
|
||||
);
|
||||
|
||||
@@ -64,7 +64,7 @@ alertActionsRouter.get("/:token", asyncHandler(async (req, res) => {
|
||||
continue;
|
||||
}
|
||||
const ack = await loadAck(alert.id, now);
|
||||
const state = alert.snoozed ? `Snoozed until ${esc(new Date(alert.snoozed.until).toISOString().slice(0, 16).replace("T", " "))} UTC.` : ack?.active ? `Acknowledged by ${esc(ack.ackedBy)}.` : alert.silenced ? "Held back (under maintenance, or folded under something that is down)." : "Nobody has acknowledged it.";
|
||||
const state = alert.closed ? `Closed by ${esc(alert.closed.by)}.` : alert.snoozed ?`Snoozed until ${esc(new Date(alert.snoozed.until).toISOString().slice(0, 16).replace("T", " "))} UTC.` : ack?.active ? `Acknowledged by ${esc(ack.ackedBy)}.` : alert.silenced ? "Held back (under maintenance, or folded under something that is down)." : "Nobody has acknowledged it.";
|
||||
const form = (action: ActionId, label: string, secondary = false) =>
|
||||
`<form method="post" action="/api/alert-actions/${esc(req.params.token)}/${action}"><input type="hidden" name="key" value="${esc(key)}"><button type="submit"${secondary ? ' class="secondary"' : ""}>${esc(label)}</button></form>`;
|
||||
cards += `<div class="card"><div class="sev ${esc(alert.severity)}">${esc(alert.severity)}</div><div class="msg">${esc(alert.message)}</div><div class="state">${state}</div>${form("ack", "Acknowledge")}${alert.target ? form("silence", `Silence for ${SILENCE_MINUTES === 60 ? "an hour" : `${SILENCE_MINUTES} min`}`, true) : ""}${form("snooze", `Snooze until ${untilText}`, true)}<div class="small"><a href="${esc(base + (alert.link ?? "/alerts"))}">Open in Homelab Manager</a></div></div>`;
|
||||
|
||||
@@ -2,11 +2,11 @@ import { Router } from "express";
|
||||
import { z } from "zod";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { getAlerts, invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { clearSnooze, setSnooze, SnoozeError } from "../services/alertSnoozes.js";
|
||||
import { clearSnooze, MAX_CLOSE_NOTE, setClosed, setSnooze, SnoozeError } from "../services/alertSnoozes.js";
|
||||
import { keyOfAlertId } from "../services/flapGuard.js";
|
||||
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { openIncidentIds } from "../services/incidents.js";
|
||||
import { addNote, openIncidentIds } from "../services/incidents.js";
|
||||
import { buildNoiseReport, NOISE_DAYS } from "../services/alertNoise.js";
|
||||
import { ticketsForAlerts, type AlertTicketInfo } from "../services/alertTickets.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
@@ -133,3 +133,46 @@ alertsRouter.delete("/snooze", requireRole("operator"), asyncHandler(async (req,
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
const closeSchema = z.object({
|
||||
alertId: z.string().min(1).max(600),
|
||||
/** Why — shown on the alert and written into its incident. */
|
||||
note: z.string().max(MAX_CLOSE_NOTE).nullable().optional(),
|
||||
});
|
||||
|
||||
// "I have dealt with it, or I accept it": the alert is hidden with no end, until it gets worse or has gone for half an hour. The reason goes on the incident, so the history says why.
|
||||
alertsRouter.put("/close", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = closeSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Say which alert, and keep the note under " + MAX_CLOSE_NOTE + " characters." });
|
||||
const alert = (await getAlerts(false)).alerts.find((a) => a.id === parsed.data.alertId);
|
||||
if (!alert) return res.status(404).json({ error: "not_found", message: "That alert isn't there any more." });
|
||||
const actor = req.currentUser!;
|
||||
const who = actor.name ?? actor.email ?? actor.oidcSub;
|
||||
try {
|
||||
const closed = await setClosed({ alertId: alert.id, conditionKey: keyOfAlertId(alert.id, alert.category), message: alert.message, severity: alert.severity, note: parsed.data.note ?? null, actor: who });
|
||||
// The incident it is part of says it was closed, by whom and why.
|
||||
const incidentId = (await openIncidentIds([alert.id])).get(alert.id);
|
||||
if (incidentId) await addNote(incidentId, who, closed.note ? "Closed: " + closed.note : "Closed without a note.");
|
||||
await recordAudit({ actor, category: "alert", action: "close", ...auditTargetOf(alert), detail: { alertId: alert.id, message: alert.message, note: closed.note } });
|
||||
invalidateAlertsCache();
|
||||
res.json({ closed: { by: closed.snoozedBy, at: closed.snoozedAt, note: closed.note } });
|
||||
} catch (err) {
|
||||
if (err instanceof SnoozeError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
// Opens it again: it is shown, counted and announced like any other (and, if it is still there, is a new message to the people who get them).
|
||||
alertsRouter.delete("/close", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const alertId = typeof req.query.alertId === "string" ? req.query.alertId : "";
|
||||
if (!alertId) return res.status(400).json({ error: "invalid_id" });
|
||||
const removed = await clearSnooze(alertId);
|
||||
if (!removed) return res.status(404).json({ error: "not_found" });
|
||||
const actor = req.currentUser!;
|
||||
const alert = (await getAlerts(false)).alerts.find((a) => a.id === alertId);
|
||||
const incidentId = alert ? (await openIncidentIds([alertId])).get(alertId) : undefined;
|
||||
if (incidentId) await addNote(incidentId, actor.name ?? actor.email ?? actor.oidcSub, "Opened again.");
|
||||
await recordAudit({ actor, category: "alert", action: "reopen", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: removed.message } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -103,7 +103,7 @@ const previousAck = z
|
||||
})
|
||||
.nullable();
|
||||
|
||||
const snoozeRecord = z.object({ conditionKey: z.string().max(700), message: z.string().max(500), until: z.string().max(40), snoozedBy: z.string().max(200), snoozedAt: z.string().max(40) });
|
||||
const snoozeRecord = z.object({ conditionKey: z.string().max(700), message: z.string().max(500), until: z.string().max(40), snoozedBy: z.string().max(200), snoozedAt: z.string().max(40), kind: z.enum(["snooze", "closed"]).optional(), severity: z.string().max(20).nullable().optional(), note: z.string().max(500).nullable().optional() });
|
||||
|
||||
const alertsBody = z.discriminatedUnion("action", [
|
||||
/** When they should come back, as a date and time with its zone. */
|
||||
|
||||
@@ -30,9 +30,17 @@ export interface SnoozeInfo {
|
||||
until: string;
|
||||
snoozedBy: string;
|
||||
snoozedAt: string;
|
||||
/** Closed rather than snoozed: it has no end, only the conditions in this file's header. */
|
||||
closed: boolean;
|
||||
note: string | null;
|
||||
}
|
||||
|
||||
const shape = (r: typeof alertSnoozes.$inferSelect): SnoozeInfo => ({ until: r.until, snoozedBy: r.snoozedBy, snoozedAt: r.snoozedAt });
|
||||
const shape = (r: typeof alertSnoozes.$inferSelect): SnoozeInfo => ({ until: r.until, snoozedBy: r.snoozedBy, snoozedAt: r.snoozedAt, closed: r.kind === "closed", note: r.note });
|
||||
|
||||
/** A closed alert has no end; this is a date so far off that it never comes, which keeps every place that asks "is it still snoozed?" working as it is. */
|
||||
export const CLOSED_UNTIL = "9999-12-31T00:00:00.000Z";
|
||||
const SEVERITY_RANK: Record<string, number> = { info: 0, warning: 1, critical: 2 };
|
||||
export const MAX_CLOSE_NOTE = 500;
|
||||
|
||||
/** The snoozes still running, by alert id. */
|
||||
export async function loadSnoozes(now: Date = new Date()): Promise<Map<string, SnoozeInfo>> {
|
||||
@@ -55,9 +63,21 @@ export async function withoutSnoozed<T extends { key?: string }>(items: T[], now
|
||||
export async function setSnooze(input: { alertId: string; conditionKey: string; message: string; until: Date; actor: string }, now: Date = new Date()): Promise<SnoozeInfo> {
|
||||
if (!(input.until.getTime() >= now.getTime() + MIN_AHEAD_MS)) throw new SnoozeError(400, "invalid_until", "Choose a time that has not come yet.");
|
||||
if (input.until.getTime() > now.getTime() + MAX_DAYS * 86_400_000) throw new SnoozeError(400, "invalid_until", `A snooze can be at most ${MAX_DAYS} days.`);
|
||||
const values = { conditionKey: input.conditionKey, message: input.message.slice(0, 500), until: input.until.toISOString(), snoozedBy: input.actor, snoozedAt: now.toISOString(), lastSeenAt: now.toISOString() };
|
||||
const values = { conditionKey: input.conditionKey, message: input.message.slice(0, 500), until: input.until.toISOString(), snoozedBy: input.actor, snoozedAt: now.toISOString(), lastSeenAt: now.toISOString(), kind: "snooze" as const, severity: null, note: null };
|
||||
await db.insert(alertSnoozes).values({ alertId: input.alertId, ...values }).onConflictDoUpdate({ target: alertSnoozes.alertId, set: values });
|
||||
return { until: values.until, snoozedBy: values.snoozedBy, snoozedAt: values.snoozedAt };
|
||||
return { until: values.until, snoozedBy: values.snoozedBy, snoozedAt: values.snoozedAt, closed: false, note: null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes an alert: "I have dealt with it, or I accept it". It is hidden like a snoozed one, with no end — until it **gets worse** (a more serious severity than it had when it was
|
||||
* closed) or has been **gone for half an hour** (the same problem coming back later is new). A number in its message changing (a disk at 91 %, then 92 %) is not a change. `note` says why.
|
||||
*/
|
||||
export async function setClosed(input: { alertId: string; conditionKey: string; message: string; severity: string; note: string | null; actor: string }, now: Date = new Date()): Promise<SnoozeInfo> {
|
||||
const note = (input.note ?? "").trim();
|
||||
if (note.length > MAX_CLOSE_NOTE) throw new SnoozeError(400, "note_too_long", `Keep the note under ${MAX_CLOSE_NOTE} characters.`);
|
||||
const values = { conditionKey: input.conditionKey, message: input.message.slice(0, 500), until: CLOSED_UNTIL, snoozedBy: input.actor, snoozedAt: now.toISOString(), lastSeenAt: now.toISOString(), kind: "closed" as const, severity: input.severity, note: note === "" ? null : note };
|
||||
await db.insert(alertSnoozes).values({ alertId: input.alertId, ...values }).onConflictDoUpdate({ target: alertSnoozes.alertId, set: values });
|
||||
return { until: values.until, snoozedBy: values.snoozedBy, snoozedAt: values.snoozedAt, closed: true, note: values.note };
|
||||
}
|
||||
|
||||
/** A snooze as it is stored, whole — what an undo puts back. */
|
||||
@@ -67,12 +87,16 @@ export interface SnoozeRecord {
|
||||
until: string;
|
||||
snoozedBy: string;
|
||||
snoozedAt: string;
|
||||
/** Left out of an older record: a snooze. */
|
||||
kind?: "snooze" | "closed";
|
||||
severity?: string | null;
|
||||
note?: string | null;
|
||||
}
|
||||
|
||||
/** The snooze on an alert as it stands, expired or not; null when it has none. */
|
||||
/** The snooze (or the closing) on an alert as it stands, expired or not; null when it has none. */
|
||||
export async function readSnooze(alertId: string): Promise<SnoozeRecord | null> {
|
||||
const [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, alertId)).limit(1);
|
||||
return row ? { conditionKey: row.conditionKey, message: row.message, until: row.until, snoozedBy: row.snoozedBy, snoozedAt: row.snoozedAt } : null;
|
||||
return row ? { conditionKey: row.conditionKey, message: row.message, until: row.until, snoozedBy: row.snoozedBy, snoozedAt: row.snoozedAt, kind: row.kind, severity: row.severity, note: row.note } : null;
|
||||
}
|
||||
|
||||
/** Puts a snooze back as it was (an undo). One whose time has already passed is not worth restoring: it would only announce the alert again at once, so it is dropped. */
|
||||
@@ -81,7 +105,7 @@ export async function restoreSnooze(alertId: string, record: SnoozeRecord, now:
|
||||
await clearSnooze(alertId);
|
||||
return;
|
||||
}
|
||||
const values = { ...record, lastSeenAt: now.toISOString() };
|
||||
const values = { ...record, kind: record.kind ?? ("snooze" as const), severity: record.severity ?? null, note: record.note ?? null, lastSeenAt: now.toISOString() };
|
||||
await db.insert(alertSnoozes).values({ alertId, ...values }).onConflictDoUpdate({ target: alertSnoozes.alertId, set: values });
|
||||
}
|
||||
|
||||
@@ -96,11 +120,14 @@ export interface Reminder {
|
||||
key: string;
|
||||
message: string;
|
||||
severity?: string;
|
||||
/** "ended": a snooze's time is up. "worse": a closed alert got more serious. */
|
||||
reason?: "ended" | "worse";
|
||||
}
|
||||
|
||||
/**
|
||||
* One look, with the alerts that are there now: snoozes that have run out are removed, and each whose alert is still there becomes a reminder; the others' alerts are
|
||||
* marked as seen, and those that have been gone for half an hour are dropped. Returns the reminders to send.
|
||||
* marked as seen, and those that have been gone for half an hour are dropped. A closed alert that is there with a higher severity than when it was closed is opened again,
|
||||
* and announced as having got worse. Returns the reminders to send.
|
||||
*/
|
||||
export async function settleSnoozes(present: { id: string; message: string; severity: string; key: string }[], now: Date = new Date()): Promise<Reminder[]> {
|
||||
const rows = await db.select().from(alertSnoozes);
|
||||
@@ -112,8 +139,11 @@ export async function settleSnoozes(present: { id: string; message: string; seve
|
||||
const cutoff = new Date(now.getTime() - GONE_AFTER_MS).toISOString();
|
||||
for (const r of rows) {
|
||||
const there = byId.get(r.alertId);
|
||||
if (Date.parse(r.until) <= now.getTime()) {
|
||||
if (there) reminders.push({ key: r.conditionKey, message: there.message, severity: there.severity });
|
||||
if (r.kind === "closed" && there && (SEVERITY_RANK[there.severity] ?? 0) > (SEVERITY_RANK[r.severity ?? ""] ?? 0)) {
|
||||
reminders.push({ key: r.conditionKey, message: there.message, severity: there.severity, reason: "worse" });
|
||||
done.push(r.id);
|
||||
} else if (Date.parse(r.until) <= now.getTime()) {
|
||||
if (there) reminders.push({ key: r.conditionKey, message: there.message, severity: there.severity, reason: "ended" });
|
||||
done.push(r.id);
|
||||
} else if (there) seen.push(r.id);
|
||||
else if (r.lastSeenAt < cutoff) done.push(r.id);
|
||||
|
||||
@@ -25,6 +25,8 @@ export interface Alert {
|
||||
affects?: string[];
|
||||
/** Snoozed until then (services/alertSnoozes.ts): held back like one under maintenance, and announced again if it is still there. */
|
||||
snoozed?: { until: string; by: string };
|
||||
/** Closed by somebody (services/alertSnoozes.ts): held back like a snoozed one, with no end — until it gets worse, or has gone for half an hour. */
|
||||
closed?: { by: string; at: string; note: string | null };
|
||||
/** Raised and cleared over and over: its raised and cleared messages are being held back (services/flapGuard.ts). */
|
||||
flapping?: boolean;
|
||||
/** What a maintenance window would have to be on to silence it, when there is such a thing — which is what "Silence" acts on. */
|
||||
|
||||
@@ -684,12 +684,13 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (flapping.size > 0) for (const a of alerts) if (flapping.has(keyOfAlertId(a.id, a.category))) a.flapping = true;
|
||||
|
||||
// Those somebody has snoozed are held back like those under maintenance, with when they come back.
|
||||
const snoozes = await loadSnoozes().catch(() => new Map<string, { until: string; snoozedBy: string }>());
|
||||
const snoozes = await loadSnoozes().catch(() => new Map<string, { until: string; snoozedBy: string; snoozedAt: string; closed: boolean; note: string | null }>());
|
||||
if (snoozes.size > 0) {
|
||||
for (const a of alerts) {
|
||||
const sn = snoozes.get(a.id);
|
||||
if (sn) {
|
||||
a.snoozed = { until: sn.until, by: sn.snoozedBy };
|
||||
if (sn.closed) a.closed = { by: sn.snoozedBy, at: sn.snoozedAt, note: sn.note };
|
||||
else a.snoozed = { until: sn.until, by: sn.snoozedBy };
|
||||
a.silenced = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,6 +223,10 @@ export function describeChange(category: string, action: string, detail: Record<
|
||||
return { title: `Alert rule removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "alert/snooze":
|
||||
return { title: `Alert snoozed${typeof detail.message === "string" ? `: ${detail.message}` : ""}`, tone: "neutral", detail: str(detail.until) ? `until ${str(detail.until)}` : null };
|
||||
case "alert/close":
|
||||
return { title: `Alert closed${typeof detail.message === "string" ? `: ${detail.message}` : ""}`, tone: "neutral", detail: str(detail.note) };
|
||||
case "alert/reopen":
|
||||
return { title: `Closed alert opened again${typeof detail.message === "string" ? `: ${detail.message}` : ""}`, tone: "neutral", detail: null };
|
||||
case "alert/unsnooze":
|
||||
return { title: `Snooze ended early${typeof detail.message === "string" ? `: ${detail.message}` : ""}`, tone: "neutral", detail: null };
|
||||
case "alert/flapping_started":
|
||||
|
||||
@@ -517,6 +517,15 @@ export async function notifySnoozeEnded(items: { message: string; key?: string;
|
||||
}), (i) => i.key);
|
||||
}
|
||||
|
||||
/** Alerts that somebody closed and that have since got more serious: opened again, and said once. */
|
||||
export async function notifyClosedWorse(items: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||||
if (items.length === 0) return;
|
||||
await notifyGrouped(items, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Closed alert got worse",
|
||||
message: `${group.length === 1 ? "An alert you closed is" : `${group.length} alerts you closed are`} more serious now, so ${group.length === 1 ? "it is" : "they are"} open again:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`,
|
||||
}), (i) => i.key);
|
||||
}
|
||||
|
||||
/** An alert that has just been called flapping: said once, with how often it has come, instead of each time. */
|
||||
export async function notifyFlapping(items: { message: string; key?: string; severity?: string; count: number }[]): Promise<void> {
|
||||
if (items.length === 0) return;
|
||||
|
||||
@@ -100,6 +100,8 @@ export async function alertsV1() {
|
||||
flapping: !!a.flapping,
|
||||
// Snoozed alerts are also `silenced`; this says until when they are.
|
||||
snoozedUntil: a.snoozed?.until ?? null,
|
||||
// Closed ones are `silenced` too; this says by whom and why. They have no end.
|
||||
closed: a.closed ? { by: a.closed.by, at: a.closed.at, note: a.closed.note } : null,
|
||||
target: a.target,
|
||||
causedBy: a.causedBy ?? null,
|
||||
affects: a.affects ?? [],
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import { getAlerts, invalidateAlertsCache } from "./alerts.js";
|
||||
import { keyOfAlertId } from "./flapGuard.js";
|
||||
import { settleSnoozes } from "./alertSnoozes.js";
|
||||
import { notifySnoozeEnded } from "./notify.js";
|
||||
import { notifyClosedWorse, notifySnoozeEnded } from "./notify.js";
|
||||
|
||||
/** One look: returns how many reminders were sent. */
|
||||
export async function remindAboutSnoozed(now: Date = new Date()): Promise<number> {
|
||||
@@ -13,7 +13,8 @@ export async function remindAboutSnoozed(now: Date = new Date()): Promise<number
|
||||
if (reminders.length > 0) {
|
||||
// The page should show them again at once, not when its stored result expires.
|
||||
invalidateAlertsCache();
|
||||
await notifySnoozeEnded(reminders);
|
||||
await notifySnoozeEnded(reminders.filter((r) => r.reason !== "worse"));
|
||||
await notifyClosedWorse(reminders.filter((r) => r.reason === "worse"));
|
||||
}
|
||||
return reminders.length;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
/**
|
||||
* Closing an alert: hidden with no end, until it gets worse or has been gone for half an hour; why it was closed goes on its incident; it can be opened again; and how it sits
|
||||
* beside snoozing.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-close-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, heartbeats, alertSnoozes, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { alertsRouter } = await import("../src/routes/alerts.js");
|
||||
const { alertsBulkRouter } = await import("../src/routes/bulk.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { setClosed, setSnooze, settleSnoozes, withoutSnoozed, loadSnoozes, GONE_AFTER_MS, CLOSED_UNTIL } = await import("../src/services/alertSnoozes.js");
|
||||
const { getIncident, openIncidentIds } = await import("../src/services/incidents.js");
|
||||
const { getChangeLog } = await import("../src/services/changeLog.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
const call = async (method: string, path: string, body?: unknown) => {
|
||||
const res = await fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
return { status: res.status, body: (await res.json().catch(() => null)) as any };
|
||||
};
|
||||
const idOfHeartbeat = async (name: string) => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const a = report.alerts.find((x) => x.message.startsWith(name));
|
||||
assert.ok(a, `an alert about ${name}`);
|
||||
return a!;
|
||||
};
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
const old = "2026-01-01 00:00:00";
|
||||
for (const name of ["jobA", "jobB"]) await db.insert(heartbeats).values({ name, token: `hb_${name}`.padEnd(24, "x"), periodMinutes: 60, graceMinutes: 5, serverId: null, createdAt: old });
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/alerts/bulk", alertsBulkRouter);
|
||||
a.use("/api/alerts", alertsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/alerts`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
describe("closing through the route", () => {
|
||||
test("a closed alert is held back like a snoozed one, with who closed it and why; it is not counted as needing attention", async () => {
|
||||
const a = await idOfHeartbeat("jobA");
|
||||
assert.equal(a.silenced, false);
|
||||
const attention = (c: { critical: number; warning: number; info: number }) => c.critical + c.warning + c.info;
|
||||
const before = (await getAlerts(false)).counts;
|
||||
const r = await call("PUT", "/close", { alertId: a.id, note: " Replaced the job; the old one is retired. " });
|
||||
assert.equal(r.status, 200);
|
||||
assert.deepEqual([r.body.closed.by, r.body.closed.note], ["Olga", "Replaced the job; the old one is retired."]);
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const after = report.alerts.find((x) => x.id === a.id)!;
|
||||
assert.deepEqual([after.silenced, after.closed?.by, after.closed?.note, after.snoozed], [true, "Olga", "Replaced the job; the old one is retired.", undefined]);
|
||||
assert.equal(attention(report.counts), attention(before) - 1, "no longer among those that need a look");
|
||||
assert.equal(report.counts.silenced, before.silenced + 1, "counted as held back");
|
||||
});
|
||||
|
||||
test("the reason is written on the incident, and opening it again says so there too", async () => {
|
||||
const a = await idOfHeartbeat("jobA");
|
||||
const incidentId = (await openIncidentIds([a.id])).get(a.id)!;
|
||||
assert.ok(incidentId, "an incident is open for it");
|
||||
let notes = (await getIncident(incidentId))!.notes.map((n) => [n.author, n.body]);
|
||||
assert.deepEqual(notes, [["Olga", "Closed: Replaced the job; the old one is retired."]]);
|
||||
assert.equal((await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`)).status, 204);
|
||||
notes = (await getIncident(incidentId))!.notes.map((n) => [n.author, n.body]);
|
||||
assert.deepEqual(notes.at(-1), ["Olga", "Opened again."]);
|
||||
invalidateAlertsCache();
|
||||
const again = (await getAlerts(true)).alerts.find((x) => x.id === a.id)!;
|
||||
assert.deepEqual([again.silenced, again.closed], [false, undefined], "shown and counted again");
|
||||
});
|
||||
|
||||
test("it is in the audit log and the change log as sentences; closing without a note says so on the incident", async () => {
|
||||
const a = await idOfHeartbeat("jobB");
|
||||
assert.equal((await call("PUT", "/close", { alertId: a.id })).status, 200);
|
||||
const incidentId = (await openIncidentIds([a.id])).get(a.id)!;
|
||||
assert.deepEqual((await getIncident(incidentId))!.notes.map((n) => n.body), ["Closed without a note."]);
|
||||
await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`);
|
||||
const actions = (await db.select().from(auditLog)).map((e) => `${e.category}/${e.action}`);
|
||||
assert.ok(actions.includes("alert/close") && actions.includes("alert/reopen"));
|
||||
const titles = (await getChangeLog({ days: 1, limit: 300 })).entries.map((e) => e.title);
|
||||
assert.ok(titles.some((t) => /^Alert closed: jobA/.test(t)), "closed");
|
||||
assert.ok(titles.some((t) => /^Closed alert opened again: jobA/.test(t)), "opened again");
|
||||
});
|
||||
|
||||
test("an alert that is not there is not found, a note that is too long is refused, a viewer cannot close or open", async () => {
|
||||
assert.equal((await call("PUT", "/close", { alertId: "monitoring:nothing:1" })).status, 404);
|
||||
const a = await idOfHeartbeat("jobA");
|
||||
const long = await call("PUT", "/close", { alertId: a.id, note: "x".repeat(501) });
|
||||
assert.equal(long.status, 400);
|
||||
who = "viewer";
|
||||
assert.equal((await call("PUT", "/close", { alertId: a.id })).status, 403);
|
||||
assert.equal((await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`)).status, 403);
|
||||
who = "operator";
|
||||
assert.equal((await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`)).status, 404, "nothing to open");
|
||||
});
|
||||
|
||||
test("closing an alert that is snoozed makes it closed, and snoozing a closed one makes it a snooze again", async () => {
|
||||
const a = await idOfHeartbeat("jobA");
|
||||
const until = new Date(Date.now() + 3_600_000).toISOString();
|
||||
assert.equal((await call("PUT", "/snooze", { alertId: a.id, until })).status, 200);
|
||||
assert.equal((await call("PUT", "/close", { alertId: a.id })).status, 200);
|
||||
let [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id));
|
||||
assert.deepEqual([row.kind, row.until], ["closed", CLOSED_UNTIL]);
|
||||
assert.equal((await call("PUT", "/snooze", { alertId: a.id, until })).status, 200);
|
||||
[row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id));
|
||||
assert.deepEqual([row.kind, row.severity, row.note], ["snooze", null, null]);
|
||||
await call("DELETE", `/snooze?alertId=${encodeURIComponent(a.id)}`);
|
||||
});
|
||||
|
||||
test("a bulk snooze over a closed alert can be undone back to closed, with its note", async () => {
|
||||
const a = await idOfHeartbeat("jobA");
|
||||
await call("PUT", "/close", { alertId: a.id, note: "accepted" });
|
||||
const r = await call("POST", "/bulk", { action: "snooze", alertIds: [a.id], until: new Date(Date.now() + 3_600_000).toISOString() });
|
||||
assert.equal(r.body.changed, 1);
|
||||
assert.equal((await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id)))[0].kind, "snooze");
|
||||
await call("POST", "/bulk", r.body.undo);
|
||||
const [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id));
|
||||
assert.deepEqual([row.kind, row.note, row.until], ["closed", "accepted", CLOSED_UNTIL]);
|
||||
await call("DELETE", `/close?alertId=${encodeURIComponent(a.id)}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("until it gets worse", () => {
|
||||
const present = (id: string, severity: string, message = "disk is 91% full") => [{ id, message, severity, key: `k:${id}` }];
|
||||
const NOW = new Date();
|
||||
|
||||
test("the same alert stays closed however its message changes, and a milder severity does not open it", async () => {
|
||||
await db.delete(alertSnoozes);
|
||||
await setClosed({ alertId: "x1", conditionKey: "k:x1", message: "disk is 91% full", severity: "warning", note: null, actor: "Olga" }, NOW);
|
||||
assert.deepEqual(await settleSnoozes(present("x1", "warning", "disk is 94% full"), NOW), []);
|
||||
assert.deepEqual(await settleSnoozes(present("x1", "info", "disk is 80% full"), NOW), []);
|
||||
assert.equal((await db.select().from(alertSnoozes)).length, 1, "still closed");
|
||||
assert.equal((await loadSnoozes(NOW)).get("x1")?.closed, true);
|
||||
});
|
||||
|
||||
test("a higher severity opens it again and says so, once", async () => {
|
||||
const reminders = await settleSnoozes(present("x1", "critical", "disk is 97% full"), NOW);
|
||||
assert.deepEqual(reminders.map((r) => [r.key, r.reason, r.severity, r.message]), [["k:x1", "worse", "critical", "disk is 97% full"]]);
|
||||
assert.equal((await db.select().from(alertSnoozes)).length, 0);
|
||||
assert.deepEqual(await settleSnoozes(present("x1", "critical"), NOW), [], "nothing more to say");
|
||||
});
|
||||
|
||||
test("gone for less than half an hour it stays closed; gone for longer it is forgotten, so the same problem later is new", async () => {
|
||||
await db.delete(alertSnoozes);
|
||||
await setClosed({ alertId: "x2", conditionKey: "k:x2", message: "m", severity: "warning", note: null, actor: "Olga" }, NOW);
|
||||
await settleSnoozes([], new Date(NOW.getTime() + GONE_AFTER_MS - 60_000));
|
||||
assert.equal((await db.select().from(alertSnoozes)).length, 1);
|
||||
await settleSnoozes([], new Date(NOW.getTime() + GONE_AFTER_MS + 60_000));
|
||||
assert.equal((await db.select().from(alertSnoozes)).length, 0);
|
||||
});
|
||||
|
||||
test("a closed alert never expires like a snooze does, however much time passes while it is there", async () => {
|
||||
await db.delete(alertSnoozes);
|
||||
await setClosed({ alertId: "x3", conditionKey: "k:x3", message: "m", severity: "warning", note: null, actor: "Olga" }, NOW);
|
||||
const tenYears = new Date(NOW.getTime() + 10 * 365 * 86_400_000);
|
||||
assert.deepEqual(await settleSnoozes(present("x3", "warning"), tenYears), []);
|
||||
assert.equal((await loadSnoozes(tenYears)).has("x3"), true);
|
||||
});
|
||||
|
||||
test("messages about a closed alert are held back like a snoozed one's, and a snooze that ends still reminds as before", async () => {
|
||||
await db.delete(alertSnoozes);
|
||||
await setClosed({ alertId: "x4", conditionKey: "k:x4", message: "m", severity: "warning", note: null, actor: "Olga" }, NOW);
|
||||
await setSnooze({ alertId: "x5", conditionKey: "k:x5", message: "snoozed one", until: new Date(NOW.getTime() + 120_000), actor: "Olga" }, NOW);
|
||||
assert.deepEqual((await withoutSnoozed([{ key: "k:x4" }, { key: "k:other" }, { key: "k:x5" }], NOW)).map((i) => i.key), ["k:other"]);
|
||||
const reminders = await settleSnoozes(present("x5", "warning", "snoozed one"), new Date(NOW.getTime() + 180_000));
|
||||
assert.deepEqual(reminders.map((r) => [r.key, r.reason]), [["k:x5", "ended"]]);
|
||||
});
|
||||
});
|
||||
@@ -315,7 +315,7 @@ describe("rules, the Alerts page and the pages that show them", () => {
|
||||
const later = new Date(now.getTime() + 2 * 3600_000 + MIN);
|
||||
await setSnooze({ alertId: "backup:gone", conditionKey: "gone", message: "gone soon", until: new Date(now.getTime() + 3 * 3600_000), actor: "Olga" }, now);
|
||||
const reminders = await settleSnoozes([{ id: "disk:disk:server:1:/", message: "web: / is 96% full", severity: "warning", key: "disk:server:1:/" }], later);
|
||||
assert.deepEqual(reminders, [{ key: "disk:server:1:/", message: "web: / is 96% full", severity: "warning" }]);
|
||||
assert.deepEqual(reminders, [{ key: "disk:server:1:/", message: "web: / is 96% full", severity: "warning", reason: "ended" }]);
|
||||
assert.equal((await withoutSnoozed([{ key: "disk:server:1:/", message: "a" }], later)).length, 1);
|
||||
// One whose alert is gone, and has been for half an hour, is dropped quietly; the snooze ending for an alert that went is forgotten without a word.
|
||||
const muchLater = new Date(now.getTime() + 4 * 3600_000);
|
||||
|
||||
Reference in new issue
Block a user