Files
Homelab-manager/server/src/routes/alerts.ts
T
bobbanandClaude Sonnet 5.5 24ae8db380 Alert rules of your own, storage forecasts for Proxmox and Synology, and snooze
- Monitoring -> Alert rules: CPU, memory, disk use and disk temperature limits with a time, severity and scope (all, one server, a tag); try before saving; findings are ordinary health conditions
- Proxmox storages and Synology volumes are written down on each health pass (hourly, 400 days) and forecast like server disks: badge and history chart on the pages, alert, digest line, /metrics gauge
- Snooze an alert until a time; held back like maintenance, messages held, announced once again if still there when it ends
- Docs updated

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-20 23:38:36 +02:00

136 lines
7.5 KiB
TypeScript

import { Router } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { getAlerts, invalidateAlertsCache } from "../services/alerts.js";
import { clearSnooze, setSnooze, SnoozeError } from "../services/alertSnoozes.js";
import { keyOfAlertId } from "../services/flapGuard.js";
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
import { recordAudit } from "../services/audit.js";
import { openIncidentIds } from "../services/incidents.js";
import { buildNoiseReport, NOISE_DAYS } from "../services/alertNoise.js";
import { ticketsForAlerts, type AlertTicketInfo } from "../services/alertTickets.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const alertsRouter = Router();
alertsRouter.use(requireAuth);
// Everyone signed in can see it — it's the same server, disk and backup status the other pages already show.
// `?refresh=1` asks for a fresh check instead of a recent one.
alertsRouter.get("/", asyncHandler(async (req, res) => {
const report = await getAlerts(req.query.refresh === "1");
// The ticket opened for each alert, if there is one. Never allowed to stop the page from loading.
let tickets = new Map<string, AlertTicketInfo>();
try {
tickets = await ticketsForAlerts(report.alerts.map((a) => a.id));
} catch (err) {
console.error("[alerts] couldn't look up tickets:", err);
}
// Who has said they are on it, and the incident record each alert is part of. Neither may stop the page from loading.
let acks = new Map<string, Awaited<ReturnType<typeof loadAcks>> extends Map<string, infer V> ? V : never>();
let incidents = new Map<string, number>();
try {
acks = await loadAcks();
incidents = await openIncidentIds(report.alerts.map((a) => a.id));
} catch (err) {
console.error("[alerts] couldn't look up acknowledgements:", err);
}
const attention = attentionCounts(report.alerts, acks);
res.json({
...report,
// `counts` still counts everything, as /metrics and the wall do; `attention` leaves out what somebody has said they are dealing with.
counts: { ...report.counts, acknowledged: attention.acknowledged, attention: { critical: attention.critical, warning: attention.warning } },
alerts: report.alerts.map((a) => ({ ...a, ticket: tickets.get(a.id) ?? null, ack: acks.get(a.id) ?? null, incidentId: incidents.get(a.id) ?? null })),
});
}));
// Which alerts fire most often and how long they last, from the incident record — for everyone signed in, like the history it comes from. ?days=7|30|90|180.
alertsRouter.get("/noise", asyncHandler(async (req, res) => {
const asked = Number(req.query.days);
res.json(await buildNoiseReport((NOISE_DAYS as readonly number[]).includes(asked) ? asked : 30));
}));
// Who an alert can be handed to.
alertsRouter.get("/assignees", requireRole("operator"), asyncHandler(async (_req, res) => {
res.json({ users: await assignableUsers() });
}));
const ackSchema = z.object({
alertId: z.string().min(1).max(600),
note: z.string().max(MAX_NOTE).default(""),
assigneeId: z.number().int().positive().nullable().default(null),
/** How long it lasts; null (or left out) is until the alert is gone. */
hours: z.number().int().min(1).max(MAX_HOURS).nullable().default(null),
});
const auditTargetOf = (alert: { target: { type: string; id: number } | null }) => (alert.target ? { targetType: alert.target.type, targetId: alert.target.id } : {});
// "I'm on it": acknowledge an alert, say why, hand it to someone, and say how long it should be left alone.
alertsRouter.put("/ack", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = ackSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "Say which alert, and optionally why, who has it and for how long.", details: parsed.error.flatten() });
const alert = (await getAlerts(false)).alerts.find((a) => a.id === parsed.data.alertId);
if (!alert) return res.status(404).json({ error: "not_found", message: "That alert isn't there any more." });
const actor = req.currentUser!;
try {
const { ack, before } = await setAck({ ...parsed.data, actor: actor.name ?? actor.email ?? actor.oidcSub });
await recordAudit({
actor,
category: "alert",
action: "ack",
...auditTargetOf(alert),
detail: { alertId: alert.id, message: alert.message, note: ack.note, assignee: ack.assignee?.name ?? null, hours: parsed.data.hours, changed: before !== null },
});
res.json({ ack });
} catch (err) {
if (err instanceof AckError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
alertsRouter.delete("/ack", requireRole("operator"), asyncHandler(async (req, res) => {
const alertId = typeof req.query.alertId === "string" ? req.query.alertId : "";
if (!alertId) return res.status(400).json({ error: "invalid_id" });
const removed = await clearAck(alertId);
if (!removed) return res.status(404).json({ error: "not_found" });
const alert = (await getAlerts(false)).alerts.find((a) => a.id === alertId);
await recordAudit({ actor: req.currentUser!, category: "alert", action: "unack", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: alert?.message ?? null } });
res.status(204).end();
}));
const snoozeSchema = z.object({
alertId: z.string().min(1).max(600),
/** When it should come back, as a date and time with its zone. */
until: z.string().min(1).max(40),
});
// "Remind me then": hide an alert until a time, and announce it again then if it is still there.
alertsRouter.put("/snooze", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = snoozeSchema.safeParse(req.body);
const until = parsed.success ? new Date(parsed.data.until) : null;
if (!parsed.success || !until || Number.isNaN(until.getTime())) return res.status(400).json({ error: "invalid_body", message: "Say which alert, and when it should come back." });
const alert = (await getAlerts(false)).alerts.find((a) => a.id === parsed.data.alertId);
if (!alert) return res.status(404).json({ error: "not_found", message: "That alert isn't there any more." });
const actor = req.currentUser!;
try {
const snooze = await setSnooze({ alertId: alert.id, conditionKey: keyOfAlertId(alert.id, alert.category), message: alert.message, until, actor: actor.name ?? actor.email ?? actor.oidcSub });
await recordAudit({ actor, category: "alert", action: "snooze", ...auditTargetOf(alert), detail: { alertId: alert.id, message: alert.message, until: snooze.until } });
invalidateAlertsCache();
res.json({ snooze });
} catch (err) {
if (err instanceof SnoozeError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
alertsRouter.delete("/snooze", requireRole("operator"), asyncHandler(async (req, res) => {
const alertId = typeof req.query.alertId === "string" ? req.query.alertId : "";
if (!alertId) return res.status(400).json({ error: "invalid_id" });
const removed = await clearSnooze(alertId);
if (!removed) return res.status(404).json({ error: "not_found" });
const alert = (await getAlerts(false)).alerts.find((a) => a.id === alertId);
await recordAudit({ actor: req.currentUser!, category: "alert", action: "unsnooze", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: removed.message } });
invalidateAlertsCache();
res.status(204).end();
}));