Alert rules of your own, storage forecasts for Proxmox and Synology, and snooze

- Monitoring -> Alert rules: CPU, memory, disk use and disk temperature limits with a time, severity and scope (all, one server, a tag); try before saving; findings are ordinary health conditions
- Proxmox storages and Synology volumes are written down on each health pass (hourly, 400 days) and forecast like server disks: badge and history chart on the pages, alert, digest line, /metrics gauge
- Snooze an alert until a time; held back like maintenance, messages held, announced once again if still there when it ends
- Docs updated

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-20 23:38:36 +02:00
1 parent 788b3e073c
commit 24ae8db380
38 files changed
+7847 -8

No files matched your search

+117
View File
@@ -0,0 +1,117 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { alertRules, alertRuleMetrics, servers } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { checkedRule, computeRuleFindings, listRules, MAX_RULES, RuleError, type RuleRow } from "../services/alertRules.js";
import { parseTags } from "../services/serverTags.js";
import { asyncHandler } from "../utils/asyncHandler.js";
/** Alert rules of your own: everyone signed in sees them and what they find now; administrators make, change and remove them. */
export const alertRulesRouter = Router();
alertRulesRouter.use(requireAuth);
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
const body = z.object({
name: z.string(),
enabled: z.boolean().optional(),
metric: z.enum(alertRuleMetrics),
mount: z.string().nullable().optional(),
operator: z.enum(["above", "below"]).optional(),
threshold: z.number(),
forMinutes: z.number().optional(),
severity: z.enum(["warning", "critical"]).optional(),
serverId: z.number().int().positive().nullable().optional(),
tag: z.string().nullable().optional(),
});
const shape = (r: RuleRow, findings: Awaited<ReturnType<typeof computeRuleFindings>>, names: Map<number, string>) => ({
id: r.id,
name: r.name,
enabled: r.enabled,
metric: r.metric,
mount: r.mount,
operator: r.operator,
threshold: r.threshold,
forMinutes: r.forMinutes,
severity: r.severity,
serverId: r.serverId,
serverName: r.serverId !== null ? (names.get(r.serverId) ?? null) : null,
tag: r.tag,
createdBy: r.createdBy,
createdAt: r.createdAt,
firing: findings.filter((f) => f.ruleId === r.id).map((f) => ({ serverId: f.serverId, serverName: f.serverName, detail: f.detail, value: f.value, since: f.since === null ? null : new Date(f.since).toISOString() })),
});
alertRulesRouter.get("/", asyncHandler(async (_req, res) => {
const rules = await listRules();
const findings = await computeRuleFindings(rules);
const serverRows = await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers);
const names = new Map(serverRows.map((s) => [s.id, s.name]));
const tags = [...new Set(serverRows.flatMap((s) => parseTags(s.tags)))].sort((a, b) => a.localeCompare(b, "sv"));
res.json({ rules: rules.map((r) => shape(r, findings, names)), servers: serverRows.map((s) => ({ id: s.id, name: s.name })).sort((a, b) => a.name.localeCompare(b.name)), tags });
}));
// What a rule that has not been saved would find right now, so it can be tried before it is.
alertRulesRouter.post("/preview", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = body.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Fill in what to watch and the limit." });
try {
const checked = await checkedRule(parsed.data);
const findings = await computeRuleFindings([{ id: 0, createdAt: "", createdBy: null, ...checked, enabled: true }]);
res.json({ findings: findings.map((f) => ({ serverId: f.serverId, serverName: f.serverName, detail: f.detail, value: f.value })) });
} catch (err) {
if (err instanceof RuleError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
alertRulesRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = body.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Fill in a name, what to watch and the limit." });
try {
if ((await listRules()).length >= MAX_RULES) throw new RuleError(400, "too_many", `That is as many rules as can be kept (${MAX_RULES}).`);
const checked = await checkedRule(parsed.data);
const actor = req.currentUser!;
const [row] = await db.insert(alertRules).values({ ...checked, createdBy: actor.name ?? actor.email ?? actor.oidcSub, createdAt: new Date().toISOString() }).returning();
await recordAudit({ actor, category: "alert_rule", action: "create", targetType: "alert_rule", targetId: row.id, detail: { name: row.name, metric: row.metric, operator: row.operator, threshold: row.threshold, forMinutes: row.forMinutes, severity: row.severity } });
invalidateAlertsCache();
res.status(201).json({ rule: shape(row, [], new Map()) });
} catch (err) {
if (err instanceof RuleError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
alertRulesRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = body.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Fill in a name, what to watch and the limit." });
const [before] = await db.select().from(alertRules).where(eq(alertRules.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
try {
const checked = await checkedRule(parsed.data);
const [row] = await db.update(alertRules).set(checked).where(eq(alertRules.id, id)).returning();
await recordAudit({ actor: req.currentUser!, category: "alert_rule", action: before.enabled !== row.enabled && before.name === row.name ? (row.enabled ? "enable" : "disable") : "update", targetType: "alert_rule", targetId: id, detail: { name: row.name, metric: row.metric, operator: row.operator, threshold: row.threshold, forMinutes: row.forMinutes, severity: row.severity } });
invalidateAlertsCache();
res.json({ rule: shape(row, [], new Map()) });
} catch (err) {
if (err instanceof RuleError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
alertRulesRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [row] = await db.select().from(alertRules).where(eq(alertRules.id, id)).limit(1);
if (!row) return res.status(404).json({ error: "not_found" });
await db.delete(alertRules).where(eq(alertRules.id, id));
await recordAudit({ actor: req.currentUser!, category: "alert_rule", action: "delete", targetType: "alert_rule", targetId: id, detail: { name: row.name } });
invalidateAlertsCache();
res.status(204).end();
}));
+39 -1
View File
@@ -1,7 +1,9 @@
import { Router } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { getAlerts } from "../services/alerts.js";
import { getAlerts, invalidateAlertsCache } from "../services/alerts.js";
import { clearSnooze, setSnooze, SnoozeError } from "../services/alertSnoozes.js";
import { keyOfAlertId } from "../services/flapGuard.js";
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
import { recordAudit } from "../services/audit.js";
import { openIncidentIds } from "../services/incidents.js";
@@ -95,3 +97,39 @@ alertsRouter.delete("/ack", requireRole("operator"), asyncHandler(async (req, re
await recordAudit({ actor: req.currentUser!, category: "alert", action: "unack", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: alert?.message ?? null } });
res.status(204).end();
}));
const snoozeSchema = z.object({
alertId: z.string().min(1).max(600),
/** When it should come back, as a date and time with its zone. */
until: z.string().min(1).max(40),
});
// "Remind me then": hide an alert until a time, and announce it again then if it is still there.
alertsRouter.put("/snooze", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = snoozeSchema.safeParse(req.body);
const until = parsed.success ? new Date(parsed.data.until) : null;
if (!parsed.success || !until || Number.isNaN(until.getTime())) return res.status(400).json({ error: "invalid_body", message: "Say which alert, and when it should come back." });
const alert = (await getAlerts(false)).alerts.find((a) => a.id === parsed.data.alertId);
if (!alert) return res.status(404).json({ error: "not_found", message: "That alert isn't there any more." });
const actor = req.currentUser!;
try {
const snooze = await setSnooze({ alertId: alert.id, conditionKey: keyOfAlertId(alert.id, alert.category), message: alert.message, until, actor: actor.name ?? actor.email ?? actor.oidcSub });
await recordAudit({ actor, category: "alert", action: "snooze", ...auditTargetOf(alert), detail: { alertId: alert.id, message: alert.message, until: snooze.until } });
invalidateAlertsCache();
res.json({ snooze });
} catch (err) {
if (err instanceof SnoozeError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
alertsRouter.delete("/snooze", requireRole("operator"), asyncHandler(async (req, res) => {
const alertId = typeof req.query.alertId === "string" ? req.query.alertId : "";
if (!alertId) return res.status(400).json({ error: "invalid_id" });
const removed = await clearSnooze(alertId);
if (!removed) return res.status(404).json({ error: "not_found" });
const alert = (await getAlerts(false)).alerts.find((a) => a.id === alertId);
await recordAudit({ actor: req.currentUser!, category: "alert", action: "unsnooze", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: removed.message } });
invalidateAlertsCache();
res.status(204).end();
}));
+27
View File
@@ -0,0 +1,27 @@
import { Router } from "express";
import { requireAuth } from "../auth/middleware.js";
import { storageHistory, storageOverview } from "../services/storageHistory.js";
import { bucketPoints } from "../services/serverMetrics.js";
import { getSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
/** The pools Proxmox and Synology report, with how fast each is filling: for everyone signed in, like the pages that show the pools. */
export const storageRouter = Router();
storageRouter.use(requireAuth);
storageRouter.get("/", asyncHandler(async (_req, res) => {
const { healthChecks } = await getSettings();
res.json({ pools: await storageOverview(), forecastDays: healthChecks.diskForecastDays });
}));
// One pool's readings, for the chart: up to the 400 days that are kept, thinned to a drawable number of points.
storageRouter.get("/:id/history", asyncHandler(async (req, res) => {
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const asked = Number(req.query.days);
const days = Number.isFinite(asked) ? Math.min(400, Math.max(1, Math.round(asked))) : 30;
const to = Date.now();
const from = to - days * 86_400_000;
const rows = await storageHistory(id, days, new Date(to));
res.json({ days, points: bucketPoints(rows.map((r) => ({ t: Date.parse(r.t), v: r.v })), from, to, 120) });
}));