Alert rules of your own, storage forecasts for Proxmox and Synology, and snooze

- Monitoring -> Alert rules: CPU, memory, disk use and disk temperature limits with a time, severity and scope (all, one server, a tag); try before saving; findings are ordinary health conditions
- Proxmox storages and Synology volumes are written down on each health pass (hourly, 400 days) and forecast like server disks: badge and history chart on the pages, alert, digest line, /metrics gauge
- Snooze an alert until a time; held back like maintenance, messages held, announced once again if still there when it ends
- Docs updated

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-20 23:38:36 +02:00
1 parent 788b3e073c
commit 24ae8db380
38 files changed
+7847 -8

No files matched your search

+410
View File
@@ -0,0 +1,410 @@
import { useCallback, useEffect, useState } from "react";
import { Link } from "react-router-dom";
import { api, type AlertRule, type AlertRuleInput, type AlertRuleMetric, type CurrentUser } from "../api/client";
import DetailDrawer from "../components/DetailDrawer";
import EmptyState from "../components/EmptyState";
import { ErrorNotice } from "../components/Notice";
import { BlockSkeleton } from "../components/Skeleton";
import TimeAgo from "../components/TimeAgo";
import { readableError } from "../utils/errors";
import { toast } from "../utils/toast";
import { removeWithUndo } from "../utils/undoable";
const METRICS: { value: AlertRuleMetric; label: string; unit: string; hint: string }[] = [
{ value: "cpu", label: "CPU load", unit: "%", hint: "As each agent reports it (load average over cores)." },
{ value: "mem", label: "Memory in use", unit: "%", hint: "The share of memory in use." },
{ value: "disk", label: "Disk use", unit: "%", hint: "How full a disk is — every disk, or the one you name." },
{ value: "disk_temp", label: "Disk temperature", unit: "°C", hint: "From the disk's own report (SMART); judged as of the last report." },
];
const metricOf = (m: AlertRuleMetric) => METRICS.find((x) => x.value === m)!;
const MINUTES: { value: number; label: string }[] = [
{ value: 0, label: "right away" },
{ value: 15, label: "15 minutes" },
{ value: 30, label: "30 minutes" },
{ value: 60, label: "1 hour" },
{ value: 120, label: "2 hours" },
{ value: 360, label: "6 hours" },
{ value: 1440, label: "a day" },
];
const minutesWords = (m: number) => MINUTES.find((x) => x.value === m)?.label ?? (m < 60 ? `${m} minutes` : `${Math.round((m / 60) * 10) / 10} hours`);
/** The rule in a sentence: what it watches, the limit, for how long and on which servers. */
function sentence(r: Pick<AlertRuleInput, "metric" | "mount" | "operator" | "threshold" | "forMinutes"> & { serverName?: string | null; tag?: string | null }): string {
const m = metricOf(r.metric);
const what = r.metric === "disk" && r.mount ? `Use of ${r.mount}` : m.label;
const forHow = r.forMinutes ? `, for ${minutesWords(r.forMinutes)}` : "";
const where = r.serverName ? `on ${r.serverName}` : r.tag ? `on servers tagged ${r.tag}` : "on every server";
return `${what} ${r.operator ?? "above"} ${r.threshold}${m.unit === "°C" ? " °C" : "%"}${forHow} — ${where}`;
}
const EXAMPLES: AlertRuleInput[] = [
{ name: "CPU above 90% for 15 minutes", metric: "cpu", threshold: 90, forMinutes: 15, severity: "warning" },
{ name: "Memory above 85% for 15 minutes", metric: "mem", threshold: 85, forMinutes: 15, severity: "warning" },
{ name: "A disk warmer than 55 °C", metric: "disk_temp", threshold: 55, severity: "warning" },
];
function RuleDrawer({ initial, servers, tags, onClose, onSaved }: { initial: AlertRuleInput & { id?: number }; servers: { id: number; name: string }[]; tags: string[]; onClose: () => void; onSaved: () => void }) {
const [name, setName] = useState(initial.name);
const [metric, setMetric] = useState<AlertRuleMetric>(initial.metric);
const [mount, setMount] = useState(initial.mount ?? "");
const [operator, setOperator] = useState<"above" | "below">(initial.operator ?? "above");
const [threshold, setThreshold] = useState(String(initial.threshold));
const [forMinutes, setForMinutes] = useState(initial.forMinutes ?? 0);
const [severity, setSeverity] = useState<"warning" | "critical">(initial.severity ?? "warning");
const [scope, setScope] = useState<"all" | "server" | "tag">(initial.serverId ? "server" : initial.tag ? "tag" : "all");
const [serverId, setServerId] = useState(initial.serverId ? String(initial.serverId) : "");
const [tag, setTag] = useState(initial.tag ?? "");
const [enabled] = useState(initial.enabled ?? true);
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
const [tried, setTried] = useState<{ serverName: string; detail: string | null; value: number }[] | null>(null);
const m = metricOf(metric);
const body = (): AlertRuleInput | null => {
const limit = Number(threshold);
if (threshold.trim() === "" || !Number.isFinite(limit)) {
setError("The limit has to be a number.");
return null;
}
return {
name: name.trim(),
enabled,
metric,
mount: metric === "disk" ? mount.trim() || null : null,
operator,
threshold: limit,
forMinutes: metric === "disk_temp" ? 0 : forMinutes,
severity,
serverId: scope === "server" && serverId ? Number(serverId) : null,
tag: scope === "tag" && tag.trim() ? tag.trim() : null,
};
};
async function tryIt() {
const b = body();
if (!b) return;
setBusy(true);
setError(null);
try {
setTried((await api.alertRules.preview({ ...b, name: b.name || "Try" })).findings);
} catch (err) {
setError(readableError(err));
} finally {
setBusy(false);
}
}
async function save() {
const b = body();
if (!b) return;
setBusy(true);
setError(null);
try {
if (initial.id) await api.alertRules.update(initial.id, b);
else await api.alertRules.create(b);
toast.success(initial.id ? "Saved." : "Rule added.");
onSaved();
onClose();
} catch (err) {
setError(readableError(err));
} finally {
setBusy(false);
}
}
return (
<DetailDrawer title={initial.id ? "Change the rule" : "Add a rule"} subtitle={initial.id ? initial.name : "A limit of your own"} onClose={onClose}>
<ErrorNotice error={error} />
<div className="mb-3">
<label className="form-label" htmlFor="rule-name">
Name
</label>
<input id="rule-name" className="form-control" maxLength={80} value={name} onChange={(e) => setName(e.target.value)} placeholder="e.g. CPU above 90% for 15 minutes" autoFocus />
<div className="form-text">It is in the alert&rsquo;s message, in square brackets.</div>
</div>
<div className="mb-3">
<label className="form-label" htmlFor="rule-metric">
Watch
</label>
<select id="rule-metric" className="form-select" value={metric} onChange={(e) => { setMetric(e.target.value as AlertRuleMetric); setTried(null); }}>
{METRICS.map((x) => (
<option key={x.value} value={x.value}>
{x.label}
</option>
))}
</select>
<div className="form-text">{m.hint}</div>
</div>
{metric === "disk" && (
<div className="mb-3">
<label className="form-label" htmlFor="rule-mount">
Which disk <span className="text-secondary">(optional)</span>
</label>
<input id="rule-mount" className="form-control" value={mount} onChange={(e) => setMount(e.target.value)} placeholder="every disk — or a mount point like /data, or C:" />
</div>
)}
<div className="row g-2 mb-3">
<div className="col-6">
<label className="form-label" htmlFor="rule-operator">
Alert when
</label>
<select id="rule-operator" className="form-select" value={operator} onChange={(e) => setOperator(e.target.value as "above" | "below")}>
<option value="above">above</option>
<option value="below">below</option>
</select>
</div>
<div className="col-6">
<label className="form-label" htmlFor="rule-threshold">
Limit ({m.unit})
</label>
<input id="rule-threshold" className="form-control" inputMode="decimal" value={threshold} onChange={(e) => { setThreshold(e.target.value); setTried(null); }} />
</div>
</div>
{metric !== "disk_temp" && (
<div className="mb-3">
<label className="form-label" htmlFor="rule-for">
For at least
</label>
<select id="rule-for" className="form-select" value={forMinutes} onChange={(e) => { setForMinutes(Number(e.target.value)); setTried(null); }}>
{MINUTES.map((x) => (
<option key={x.value} value={x.value}>
{x.label}
</option>
))}
</select>
<div className="form-text">Every reading in that time has to be past the limit. Agents report every 15 minutes or so, and rules are looked at on the same schedule.</div>
</div>
)}
<div className="row g-2 mb-3">
<div className="col-6">
<label className="form-label" htmlFor="rule-severity">
How serious
</label>
<select id="rule-severity" className="form-select" value={severity} onChange={(e) => setSeverity(e.target.value as "warning" | "critical")}>
<option value="warning">Warning</option>
<option value="critical">Critical</option>
</select>
</div>
<div className="col-6">
<label className="form-label" htmlFor="rule-scope">
For
</label>
<select id="rule-scope" className="form-select" value={scope} onChange={(e) => { setScope(e.target.value as typeof scope); setTried(null); }}>
<option value="all">Every server</option>
<option value="server">One server</option>
<option value="tag" disabled={tags.length === 0}>
Servers with a tag
</option>
</select>
</div>
</div>
{scope === "server" && (
<select className="form-select mb-3" aria-label="Which server" value={serverId} onChange={(e) => { setServerId(e.target.value); setTried(null); }}>
<option value="">Choose a server…</option>
{servers.map((s) => (
<option key={s.id} value={s.id}>
{s.name}
</option>
))}
</select>
)}
{scope === "tag" && (
<select className="form-select mb-3" aria-label="Which tag" value={tag} onChange={(e) => { setTag(e.target.value); setTried(null); }}>
<option value="">Choose a tag…</option>
{tags.map((t) => (
<option key={t} value={t}>
{t}
</option>
))}
</select>
)}
<div className="text-secondary small mb-3">
Severity and kind decide who hears of it: these are <strong>Monitoring</strong> alerts, so the routing rules under Settings → Notifications apply to them like any other.
</div>
{tried && (
<div className={`alert ${tried.length > 0 ? "alert-warning" : "alert-success"} py-2`}>
{tried.length === 0 ? (
"Right now this would not be raised on any server."
) : (
<>
Right now this would be raised on {tried.length} {tried.length === 1 ? "server" : "servers"}: {tried.slice(0, 6).map((f) => `${f.serverName}${f.detail ? ` (${f.detail})` : ""} ${Math.round(f.value)}${m.unit === "°C" ? " °C" : "%"}`).join(", ")}
{tried.length > 6 ? ` and ${tried.length - 6} more` : ""}.
</>
)}
</div>
)}
<div className="d-flex gap-2">
<button type="button" className="btn btn-primary" disabled={busy || name.trim() === ""} onClick={() => void save()}>
{busy ? "Saving…" : "Save"}
</button>
<button type="button" className="btn" disabled={busy} onClick={() => void tryIt()}>
Try it now
</button>
</div>
</DetailDrawer>
);
}
/**
* Limits of your own: "CPU above 90% for 15 minutes", "memory above 85%", "a disk warmer than 55 °C". Each is judged for the servers it covers on the same schedule as the built-in
* checks, so what it finds is an ordinary alert: announced, recovered, silenced by maintenance and routed like the rest. Everyone signed in sees them; administrators change them.
*/
export default function AlertRules({ user }: { user: CurrentUser }) {
const canEdit = user.role === "admin";
const [data, setData] = useState<{ rules: AlertRule[]; servers: { id: number; name: string }[]; tags: string[] } | null>(null);
const [error, setError] = useState<string | null>(null);
const [editing, setEditing] = useState<(AlertRuleInput & { id?: number }) | null>(null);
const load = useCallback(
() =>
api.alertRules
.list()
.then((res) => {
setData(res);
setError(null);
})
.catch((err) => setError(readableError(err))),
[],
);
useEffect(() => {
void load();
}, [load]);
const asInput = (r: AlertRule): AlertRuleInput & { id: number } => ({ id: r.id, name: r.name, enabled: r.enabled, metric: r.metric, mount: r.mount, operator: r.operator, threshold: r.threshold, forMinutes: r.forMinutes, severity: r.severity, serverId: r.serverId, tag: r.tag });
async function toggle(r: AlertRule) {
try {
const { id, ...rest } = asInput(r);
await api.alertRules.update(id, { ...rest, enabled: !r.enabled });
toast.success(r.enabled ? "Switched off." : "Switched on.");
await load();
} catch (err) {
toast.error(readableError(err));
}
}
async function remove(r: AlertRule) {
const { id, ...rest } = asInput(r);
await removeWithUndo({
remove: () => api.alertRules.remove(id),
restore: () => api.alertRules.create(rest),
message: `Removed the rule "${r.name}".`,
reload: () => void load(),
});
}
return (
<>
<div className="d-flex flex-wrap align-items-center gap-2 mb-1">
<h2 className="page-title mb-0">Alert rules</h2>
{canEdit && (
<div className="ms-auto">
<button className="btn btn-primary" onClick={() => setEditing({ name: "", metric: "cpu", threshold: 90, forMinutes: 15, severity: "warning" })}>
Add a rule
</button>
</div>
)}
</div>
<div className="text-secondary mb-3">
Limits of your own, on top of the built-in checks: &ldquo;CPU above 90% for 15 minutes&rdquo;, &ldquo;memory above 85%&rdquo;, &ldquo;a disk warmer than 55 °C&rdquo;. What a rule finds is an ordinary alert on the{" "}
<Link to="/alerts">Alerts page</Link> — raised and cleared with a message, held back by maintenance, routed by the notification rules. A server that is not reporting is not judged; its own offline alert says that.
</div>
<ErrorNotice error={error} />
{data === null && !error && <BlockSkeleton lines={3} />}
{data && data.rules.length === 0 && (
<div className="card">
<EmptyState
title="No rules of your own yet"
hint={canEdit ? "Start from one of these, or add your own." : "An administrator can add them."}
action={canEdit ? { label: "Add a rule", onClick: () => setEditing({ name: "", metric: "cpu", threshold: 90, forMinutes: 15, severity: "warning" }) } : undefined}
/>
{canEdit && (
<div className="d-flex flex-wrap justify-content-center gap-2 pb-4">
{EXAMPLES.map((e) => (
<button key={e.name} className="btn btn-sm btn-outline-secondary" onClick={() => setEditing(e)}>
{e.name}
</button>
))}
</div>
)}
</div>
)}
{data && data.rules.length > 0 && (
<div className="card">
<div className="table-responsive">
<table className="table table-vcenter card-table table-stack">
<thead>
<tr>
<th>Rule</th>
<th>Severity</th>
<th>Right now</th>
{canEdit && (
<th className="w-1">
<span className="visually-hidden">Actions</span>
</th>
)}
</tr>
</thead>
<tbody>
{data.rules.map((r) => (
<tr key={r.id} className={r.enabled ? "" : "text-secondary"}>
<td className="stack-wide">
<strong>{r.name}</strong>
{!r.enabled && <span className="badge bg-secondary-lt text-secondary ms-2">off</span>}
<div className="text-secondary small">{sentence(r)}</div>
</td>
<td className="stack-label" data-label="Severity">
<span className={`badge ${r.severity === "critical" ? "bg-red-lt text-red" : "bg-yellow-lt text-yellow"}`}>{r.severity === "critical" ? "Critical" : "Warning"}</span>
</td>
<td className="stack-label" data-label="Right now">
{!r.enabled ? (
<span className="text-secondary">—</span>
) : r.firing.length === 0 ? (
<span className="text-green">Not raised</span>
) : (
<div>
{r.firing.slice(0, 4).map((f, i) => (
<div key={i} className="small">
<Link to={`/servers/${f.serverId}?view=history`}>{f.serverName}</Link>
{f.detail ? ` · ${f.detail}` : ""} · {Math.round(f.value)}
{r.metric === "disk_temp" ? " °C" : "%"}
{f.since ? (
<>
{" "}
· since <TimeAgo value={f.since} />
</>
) : null}
</div>
))}
{r.firing.length > 4 && <div className="small text-secondary">and {r.firing.length - 4} more</div>}
</div>
)}
</td>
{canEdit && (
<td className="text-nowrap stack-wide">
<div className="btn-list flex-nowrap">
<button className="btn btn-sm btn-outline-secondary" onClick={() => setEditing(asInput(r))}>
Change
</button>
<button className="btn btn-sm btn-outline-secondary" onClick={() => void toggle(r)}>
{r.enabled ? "Switch off" : "Switch on"}
</button>
<button className="btn btn-sm btn-outline-danger" onClick={() => void remove(r)} aria-label={`Remove ${r.name}`}>
Remove
</button>
</div>
</td>
)}
</tr>
))}
</tbody>
</table>
</div>
</div>
)}
{editing && data && <RuleDrawer initial={editing} servers={data.servers} tags={data.tags} onClose={() => setEditing(null)} onSaved={() => void load()} />}
</>
);
}
+30 -1
View File
@@ -9,12 +9,14 @@ import { downloadCsv } from "../utils/csv";
import { formatDateTime } from "../utils/date";
import { formatAgo } from "../utils/duration";
import { readableError } from "../utils/errors";
import { toast } from "../utils/toast";
import { ErrorNotice } from "../components/Notice";
import TableToolbar from "../components/TableToolbar";
import SilenceButton from "../components/SilenceButton";
import RunPlaybookButton from "../components/RunPlaybookButton";
import RunbookEntry from "../components/RunbookEntry";
import AckDrawer, { ackWords } from "../components/AckDrawer";
import SnoozeDrawer, { snoozeWords } from "../components/SnoozeDrawer";
import { useAlertsReport } from "../hooks/useAlertsReport";
import { summariseAlerts } from "../utils/alertSummary";
import { EmptyRow } from "../components/EmptyState";
@@ -94,6 +96,7 @@ function AlertsNow({ user, tabs }: { user: CurrentUser; tabs: React.ReactNode })
// Which alerts to show by whether somebody is on them, and the one whose acknowledgement is being edited.
const [attention, setAttention] = useState<"all" | "unacknowledged" | "acknowledged" | "mine">("all");
const [acking, setAcking] = useState<AlertItem | null>(null);
const [snoozing, setSnoozing] = useState<AlertItem | null>(null);
// Alerts ticked for a bulk action, and which bulk form is open.
const [picked, setPicked] = useState<Set<string>>(new Set());
const [bulkForm, setBulkForm] = useState<"ack" | "silence" | null>(null);
@@ -195,6 +198,17 @@ function AlertsNow({ user, tabs }: { user: CurrentUser; tabs: React.ReactNode })
void load(true);
reloadBadge();
}
async function wake(a: AlertItem) {
try {
await api.alerts.unsnooze(a.id);
toast.success("Awake again.");
await load(true);
reloadBadge();
} catch (err) {
toast.error(readableError(err));
}
}
async function bulkAck(v: { note: string; assigneeId: number | null; hours: number | null }) {
const ok = await doBulk({ call: api.bulk.alerts, body: { action: "acknowledge", alertIds: pickedIds, ...v }, done: (r) => `${plural(r.changed, "alert")} acknowledged.`, reload: afterBulk });
if (ok) {
@@ -441,6 +455,10 @@ function AlertsNow({ user, tabs }: { user: CurrentUser; tabs: React.ReactNode })
<span className="badge bg-secondary-lt text-secondary ms-2" title={`${a.causedBy.name} is down too, which explains this — notifications for this are held back. If ${a.causedBy.name} comes back and this stays down, it alerts then.`}>
caused by {a.causedBy.name}
</span>
) : a.snoozed ? (
<span className="badge bg-azure-lt text-azure ms-2" title={`Snoozed by ${a.snoozed.by} — hidden until then, and announced again if it is still there`}>
{snoozeWords(a.snoozed)}
</span>
) : (
a.silenced && (
<span className="badge bg-secondary-lt text-secondary ms-2" title="Under a maintenance window — notifications for this are held back">
@@ -514,7 +532,17 @@ function AlertsNow({ user, tabs }: { user: CurrentUser; tabs: React.ReactNode })
{a.ack?.active ? "Acknowledged" : "Acknowledge…"}
</button>
)}
<SilenceButton alert={a} onDone={() => void load(true)} />
{!a.silenced && (
<button type="button" className="btn btn-sm btn-outline-secondary" onClick={() => setSnoozing(a)} title="Hide it until a time, and hear of it again then if it is still there">
Snooze…
</button>
)}
{a.snoozed && (
<button type="button" className="btn btn-sm btn-outline-secondary" onClick={() => void wake(a)}>
Wake now
</button>
)}
{!a.snoozed && <SilenceButton alert={a} onDone={() => void load(true)} />}
{a.target?.type === "server" && !a.silenced && <RunPlaybookButton serverName={a.target.name} />}
</div>
</td>
@@ -567,6 +595,7 @@ function AlertsNow({ user, tabs }: { user: CurrentUser; tabs: React.ReactNode })
{bulkForm === "ack" && <AckBulkDrawer count={pickedIds.length} onSubmit={bulkAck} onClose={() => setBulkForm(null)} />}
{bulkForm === "silence" && <SilenceBulkDrawer noun={pickedIds.length === 1 ? "alert" : "alerts"} count={pickedIds.length} canSilenceDependents={false} onSubmit={bulkSilence} onClose={() => setBulkForm(null)} />}
{snoozing && <SnoozeDrawer alert={snoozing} onClose={() => setSnoozing(null)} onSaved={() => { void load(true); reloadBadge(); }} />}
{acking && <AckDrawer alert={acking} me={user} onClose={() => setAcking(null)} onSaved={() => {
void load(true);
reloadBadge();
+4
View File
@@ -23,6 +23,7 @@ import { toast } from "../utils/toast";
import { SearchBox } from "../components/TableToolbar";
import { useTableSearch } from "../hooks/useTableSearch";
import DetailDrawer, { ObjectDetails } from "../components/DetailDrawer";
import { proxmoxKeys, StorageForecastBadge, useStoragePools } from "../components/StorageForecast";
import { useRowDetail } from "../hooks/useRowDetail";
import { formatDateTime } from "../utils/date";
import ExportTableButton from "../components/ExportTableButton";
@@ -124,6 +125,8 @@ export default function Proxmox({ user }: { user: CurrentUser }) {
}, []);
const selected = integrations?.find((i) => i.id === selectedId) ?? null;
const { pools, forecastDays } = useStoragePools();
const poolOf = (key: string) => pools.find((p) => p.integrationId === selected?.id && p.key === key);
function loadGuests(id: number) {
setLoading(true);
@@ -325,6 +328,7 @@ export default function Proxmox({ user }: { user: CurrentUser }) {
<td>
{formatBytes(s.usedBytes)} / {formatBytes(s.totalBytes)}
{usageBar(s.usedBytes, s.totalBytes)}
{s.active && <div className="small"><StorageForecastBadge pool={poolOf(proxmoxKeys.storage(n.node, s.id, s.shared))} forecastDays={forecastDays} /></div>}
</td>
</tr>
))}
+5
View File
@@ -1,4 +1,5 @@
import { useEffect, useState } from "react";
import { StorageForecastBadge, synologyKey, useStoragePools } from "../components/StorageForecast";
import { Link } from "react-router-dom";
import {
api,
@@ -79,6 +80,7 @@ export default function Synology({ user: _user }: { user: CurrentUser }) {
}, []);
const selected = integrations?.find((i) => i.id === selectedId) ?? null;
const { pools, forecastDays } = useStoragePools();
function loadStorage(id: number) {
setLoading(true);
@@ -298,6 +300,9 @@ export default function Synology({ user: _user }: { user: CurrentUser }) {
<td className="text-secondary">{v.deviceType}</td>
<td className="text-secondary">
{formatBytes(v.sizeUsed)} / {formatBytes(v.sizeTotal)}
<div className="small">
<StorageForecastBadge pool={pools.find((p) => p.integrationId === selected?.id && p.key === synologyKey(v.id))} forecastDays={forecastDays} />
</div>
</td>
<td>{healthBadge(v.status)}</td>
</tr>