Alert rules of your own, storage forecasts for Proxmox and Synology, and snooze
- Monitoring -> Alert rules: CPU, memory, disk use and disk temperature limits with a time, severity and scope (all, one server, a tag); try before saving; findings are ordinary health conditions - Proxmox storages and Synology volumes are written down on each health pass (hourly, 400 days) and forecast like server disks: badge and history chart on the pages, alert, digest line, /metrics gauge - Snooze an alert until a time; held back like maintenance, messages held, announced once again if still there when it ends - Docs updated Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
788b3e073c
commit
24ae8db380
38 files changed
+7847
-8
No files matched your search
@@ -0,0 +1,117 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { alertRules, alertRuleMetrics, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { checkedRule, computeRuleFindings, listRules, MAX_RULES, RuleError, type RuleRow } from "../services/alertRules.js";
|
||||
import { parseTags } from "../services/serverTags.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
/** Alert rules of your own: everyone signed in sees them and what they find now; administrators make, change and remove them. */
|
||||
export const alertRulesRouter = Router();
|
||||
alertRulesRouter.use(requireAuth);
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
const body = z.object({
|
||||
name: z.string(),
|
||||
enabled: z.boolean().optional(),
|
||||
metric: z.enum(alertRuleMetrics),
|
||||
mount: z.string().nullable().optional(),
|
||||
operator: z.enum(["above", "below"]).optional(),
|
||||
threshold: z.number(),
|
||||
forMinutes: z.number().optional(),
|
||||
severity: z.enum(["warning", "critical"]).optional(),
|
||||
serverId: z.number().int().positive().nullable().optional(),
|
||||
tag: z.string().nullable().optional(),
|
||||
});
|
||||
|
||||
const shape = (r: RuleRow, findings: Awaited<ReturnType<typeof computeRuleFindings>>, names: Map<number, string>) => ({
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
enabled: r.enabled,
|
||||
metric: r.metric,
|
||||
mount: r.mount,
|
||||
operator: r.operator,
|
||||
threshold: r.threshold,
|
||||
forMinutes: r.forMinutes,
|
||||
severity: r.severity,
|
||||
serverId: r.serverId,
|
||||
serverName: r.serverId !== null ? (names.get(r.serverId) ?? null) : null,
|
||||
tag: r.tag,
|
||||
createdBy: r.createdBy,
|
||||
createdAt: r.createdAt,
|
||||
firing: findings.filter((f) => f.ruleId === r.id).map((f) => ({ serverId: f.serverId, serverName: f.serverName, detail: f.detail, value: f.value, since: f.since === null ? null : new Date(f.since).toISOString() })),
|
||||
});
|
||||
|
||||
alertRulesRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const rules = await listRules();
|
||||
const findings = await computeRuleFindings(rules);
|
||||
const serverRows = await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers);
|
||||
const names = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
const tags = [...new Set(serverRows.flatMap((s) => parseTags(s.tags)))].sort((a, b) => a.localeCompare(b, "sv"));
|
||||
res.json({ rules: rules.map((r) => shape(r, findings, names)), servers: serverRows.map((s) => ({ id: s.id, name: s.name })).sort((a, b) => a.name.localeCompare(b.name)), tags });
|
||||
}));
|
||||
|
||||
// What a rule that has not been saved would find right now, so it can be tried before it is.
|
||||
alertRulesRouter.post("/preview", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = body.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Fill in what to watch and the limit." });
|
||||
try {
|
||||
const checked = await checkedRule(parsed.data);
|
||||
const findings = await computeRuleFindings([{ id: 0, createdAt: "", createdBy: null, ...checked, enabled: true }]);
|
||||
res.json({ findings: findings.map((f) => ({ serverId: f.serverId, serverName: f.serverName, detail: f.detail, value: f.value })) });
|
||||
} catch (err) {
|
||||
if (err instanceof RuleError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
alertRulesRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = body.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Fill in a name, what to watch and the limit." });
|
||||
try {
|
||||
if ((await listRules()).length >= MAX_RULES) throw new RuleError(400, "too_many", `That is as many rules as can be kept (${MAX_RULES}).`);
|
||||
const checked = await checkedRule(parsed.data);
|
||||
const actor = req.currentUser!;
|
||||
const [row] = await db.insert(alertRules).values({ ...checked, createdBy: actor.name ?? actor.email ?? actor.oidcSub, createdAt: new Date().toISOString() }).returning();
|
||||
await recordAudit({ actor, category: "alert_rule", action: "create", targetType: "alert_rule", targetId: row.id, detail: { name: row.name, metric: row.metric, operator: row.operator, threshold: row.threshold, forMinutes: row.forMinutes, severity: row.severity } });
|
||||
invalidateAlertsCache();
|
||||
res.status(201).json({ rule: shape(row, [], new Map()) });
|
||||
} catch (err) {
|
||||
if (err instanceof RuleError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
alertRulesRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = body.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Fill in a name, what to watch and the limit." });
|
||||
const [before] = await db.select().from(alertRules).where(eq(alertRules.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
try {
|
||||
const checked = await checkedRule(parsed.data);
|
||||
const [row] = await db.update(alertRules).set(checked).where(eq(alertRules.id, id)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "alert_rule", action: before.enabled !== row.enabled && before.name === row.name ? (row.enabled ? "enable" : "disable") : "update", targetType: "alert_rule", targetId: id, detail: { name: row.name, metric: row.metric, operator: row.operator, threshold: row.threshold, forMinutes: row.forMinutes, severity: row.severity } });
|
||||
invalidateAlertsCache();
|
||||
res.json({ rule: shape(row, [], new Map()) });
|
||||
} catch (err) {
|
||||
if (err instanceof RuleError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
alertRulesRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.select().from(alertRules).where(eq(alertRules.id, id)).limit(1);
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(alertRules).where(eq(alertRules.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "alert_rule", action: "delete", targetType: "alert_rule", targetId: id, detail: { name: row.name } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -1,7 +1,9 @@
|
||||
import { Router } from "express";
|
||||
import { z } from "zod";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { getAlerts } from "../services/alerts.js";
|
||||
import { getAlerts, invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { clearSnooze, setSnooze, SnoozeError } from "../services/alertSnoozes.js";
|
||||
import { keyOfAlertId } from "../services/flapGuard.js";
|
||||
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { openIncidentIds } from "../services/incidents.js";
|
||||
@@ -95,3 +97,39 @@ alertsRouter.delete("/ack", requireRole("operator"), asyncHandler(async (req, re
|
||||
await recordAudit({ actor: req.currentUser!, category: "alert", action: "unack", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: alert?.message ?? null } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
const snoozeSchema = z.object({
|
||||
alertId: z.string().min(1).max(600),
|
||||
/** When it should come back, as a date and time with its zone. */
|
||||
until: z.string().min(1).max(40),
|
||||
});
|
||||
|
||||
// "Remind me then": hide an alert until a time, and announce it again then if it is still there.
|
||||
alertsRouter.put("/snooze", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = snoozeSchema.safeParse(req.body);
|
||||
const until = parsed.success ? new Date(parsed.data.until) : null;
|
||||
if (!parsed.success || !until || Number.isNaN(until.getTime())) return res.status(400).json({ error: "invalid_body", message: "Say which alert, and when it should come back." });
|
||||
const alert = (await getAlerts(false)).alerts.find((a) => a.id === parsed.data.alertId);
|
||||
if (!alert) return res.status(404).json({ error: "not_found", message: "That alert isn't there any more." });
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
const snooze = await setSnooze({ alertId: alert.id, conditionKey: keyOfAlertId(alert.id, alert.category), message: alert.message, until, actor: actor.name ?? actor.email ?? actor.oidcSub });
|
||||
await recordAudit({ actor, category: "alert", action: "snooze", ...auditTargetOf(alert), detail: { alertId: alert.id, message: alert.message, until: snooze.until } });
|
||||
invalidateAlertsCache();
|
||||
res.json({ snooze });
|
||||
} catch (err) {
|
||||
if (err instanceof SnoozeError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
alertsRouter.delete("/snooze", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const alertId = typeof req.query.alertId === "string" ? req.query.alertId : "";
|
||||
if (!alertId) return res.status(400).json({ error: "invalid_id" });
|
||||
const removed = await clearSnooze(alertId);
|
||||
if (!removed) return res.status(404).json({ error: "not_found" });
|
||||
const alert = (await getAlerts(false)).alerts.find((a) => a.id === alertId);
|
||||
await recordAudit({ actor: req.currentUser!, category: "alert", action: "unsnooze", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: removed.message } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -0,0 +1,27 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { storageHistory, storageOverview } from "../services/storageHistory.js";
|
||||
import { bucketPoints } from "../services/serverMetrics.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
/** The pools Proxmox and Synology report, with how fast each is filling: for everyone signed in, like the pages that show the pools. */
|
||||
export const storageRouter = Router();
|
||||
storageRouter.use(requireAuth);
|
||||
|
||||
storageRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const { healthChecks } = await getSettings();
|
||||
res.json({ pools: await storageOverview(), forecastDays: healthChecks.diskForecastDays });
|
||||
}));
|
||||
|
||||
// One pool's readings, for the chart: up to the 400 days that are kept, thinned to a drawable number of points.
|
||||
storageRouter.get("/:id/history", asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const asked = Number(req.query.days);
|
||||
const days = Number.isFinite(asked) ? Math.min(400, Math.max(1, Math.round(asked))) : 30;
|
||||
const to = Date.now();
|
||||
const from = to - days * 86_400_000;
|
||||
const rows = await storageHistory(id, days, new Date(to));
|
||||
res.json({ days, points: bucketPoints(rows.map((r) => ({ t: Date.parse(r.t), v: r.v })), from, to, 120) });
|
||||
}));
|
||||
Reference in new issue
Block a user