Alert when a server goes silent, a disk fills up, or a Synology volume degrades
The data was all being collected (agent last-seen, per-disk usage, Proxmox storage, Synology volume/disk health) but nothing acted on it, so a dead server or a full disk was only noticed by opening the right page. A new health pass runs every 15 minutes (matching the agent's default report interval) and raises one notification when a problem starts and one when it clears: a server's agent silent past a threshold (default 60 min), a server disk / Proxmox storage or root filesystem / Synology volume at or above a usage threshold (default 90%), and a Synology volume or disk that isn't "normal", has bad SMART, bad sectors past the threshold, or life remaining below it. Both thresholds and an on/off toggle live under Settings -> Notifications. The parts that make this trustworthy rather than noisy: - A problem is keyed by identity, so it alerts once and not every run; a shared Proxmox storage listed by every node is one problem, not one per node. - Active problems persist across restarts, so a rebuild doesn't re-alert everything already known. - If a source can't be read on a given run (Proxmox/Synology unreachable, one node lacking privileges) its existing problems are held, not reported "cleared" and then re-alerted when it comes back — the integration-failure alert already owns "the integration is down". - For 20 minutes after startup server-derived problems are held too: agents couldn't report while the app was down, so judging them then would report every server offline after any restart. - An offline server's disk figures are stale and are not judged; a server that never reported has no agent and raises nothing. - Tracking continues while the toggle is off (only sending is gated), so turning it back on doesn't dump every long-standing problem. Timestamps without a zone (SQLite's format) are read as UTC; the test runs on a UTC+2 machine, where reading them as local time gives a different answer. Verified with 32 checks: the evaluation rules and the state diff as pure functions (exact thresholds, the proxmox:1 vs proxmox:10 prefix trap, the flapping sequence), then a whole pass against a real Proxmox adapter talking to a fake HTTPS cluster (one node returning 403, the whole API down, a shared storage on two nodes, a node that recovers), a webhook receiver, the real DB, and the persisted state. Not exercised end-to-end: the Synology collection path — its rules are tested on data shaped exactly like the adapter's output types, but I did not stand up a fake DSM. Real dev database mtime untouched. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5 committed
1 parent
bf03a15337
commit
1688de3ea2
9 files changed
+420
-1
No files matched your search
@@ -0,0 +1,307 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, servers } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
|
||||
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
const STATE_FLAG = "healthActiveConditions";
|
||||
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
|
||||
const STARTUP_GRACE_MS = 20 * 60 * 1000;
|
||||
const PROCESS_START = Date.now();
|
||||
|
||||
// ─── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface HealthCondition {
|
||||
/** Stable identity of the problem — the same problem always produces the same key, so it alerts once, not every run. */
|
||||
key: string;
|
||||
/** Where the data came from, hierarchically ("server", "proxmox:3", "proxmox:3:pve1"). Used to hold a condition when its source can't be reached. */
|
||||
source: string;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface ServerSnapshot {
|
||||
id: number;
|
||||
name: string;
|
||||
lastSeenAt: string | null;
|
||||
disks: { mount: string; sizeBytes: number; usedBytes: number }[];
|
||||
}
|
||||
export interface ProxmoxSnapshot {
|
||||
integrationId: number;
|
||||
integrationName: string;
|
||||
nodes: ProxmoxNodeStats[];
|
||||
}
|
||||
export interface SynologySnapshot {
|
||||
integrationId: number;
|
||||
integrationName: string;
|
||||
storage: SynologyStorageInfo;
|
||||
}
|
||||
export interface HealthSnapshot {
|
||||
servers: ServerSnapshot[];
|
||||
proxmox: ProxmoxSnapshot[];
|
||||
synology: SynologySnapshot[];
|
||||
}
|
||||
export interface Thresholds {
|
||||
serverOfflineMinutes: number;
|
||||
diskUsagePercent: number;
|
||||
}
|
||||
|
||||
// ─── Formatting helpers ─────────────────────────────────────────────────────
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
const units = ["B", "KB", "MB", "GB", "TB", "PB"];
|
||||
let value = bytes;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) {
|
||||
value /= 1024;
|
||||
unit++;
|
||||
}
|
||||
return `${value.toFixed(value >= 100 || unit === 0 ? 0 : 1)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
function formatDuration(ms: number): string {
|
||||
const minutes = Math.floor(ms / 60_000);
|
||||
if (minutes < 60) return `${minutes} min`;
|
||||
const hours = Math.floor(minutes / 60);
|
||||
if (hours < 48) return `${hours} h`;
|
||||
return `${Math.floor(hours / 24)} days`;
|
||||
}
|
||||
|
||||
/** Timestamps this app writes are ISO strings, but a bare SQLite "YYYY-MM-DD HH:MM:SS" (UTC, no zone) must not be read as local time. */
|
||||
function parseTimestamp(value: string): number {
|
||||
return Date.parse(/[zZ]|[+-]\d{2}:?\d{2}$/.test(value) ? value : `${value.replace(" ", "T")}Z`);
|
||||
}
|
||||
|
||||
function usage(used: number | null, total: number | null): number | null {
|
||||
if (used === null || total === null || total <= 0) return null;
|
||||
return (used / total) * 100;
|
||||
}
|
||||
|
||||
// ─── Evaluation (pure) ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Turns a snapshot of what every source currently reports into the list of
|
||||
* problems that exist right now. Pure — no I/O, no clock other than `now` —
|
||||
* so the rules can be tested exactly.
|
||||
*/
|
||||
export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds, now: number, opts: { skipServers?: boolean } = {}): HealthCondition[] {
|
||||
const out: HealthCondition[] = [];
|
||||
const limit = thresholds.diskUsagePercent;
|
||||
const pct = (n: number) => `${Math.round(n)}%`;
|
||||
|
||||
if (!opts.skipServers) {
|
||||
for (const s of snapshot.servers) {
|
||||
let offline = false;
|
||||
if (s.lastSeenAt) {
|
||||
const age = now - parseTimestamp(s.lastSeenAt);
|
||||
if (Number.isFinite(age) && age > thresholds.serverOfflineMinutes * 60_000) {
|
||||
offline = true;
|
||||
out.push({
|
||||
key: `offline:server:${s.id}`,
|
||||
source: "server",
|
||||
message: `${s.name} hasn't reported for ${formatDuration(age)}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
// A server that never reported has no agent to go quiet; and an offline one's disk figures are stale, so neither is judged on disks.
|
||||
if (s.lastSeenAt && !offline) {
|
||||
for (const d of s.disks) {
|
||||
const p = usage(d.usedBytes, d.sizeBytes);
|
||||
if (p !== null && p >= limit) {
|
||||
out.push({
|
||||
key: `disk:server:${s.id}:${d.mount}`,
|
||||
source: "server",
|
||||
message: `${s.name}: ${d.mount} is ${pct(p)} full (${formatBytes(d.usedBytes)} of ${formatBytes(d.sizeBytes)})`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const px of snapshot.proxmox) {
|
||||
const seenShared = new Set<string>();
|
||||
for (const node of px.nodes) {
|
||||
if (node.error) continue; // couldn't be read — held by the caller, not judged
|
||||
const rootP = usage(node.rootfsUsedBytes, node.rootfsTotalBytes);
|
||||
if (rootP !== null && rootP >= limit) {
|
||||
out.push({
|
||||
key: `disk:proxmox:${px.integrationId}:${node.node}:rootfs`,
|
||||
source: `proxmox:${px.integrationId}:${node.node}`,
|
||||
message: `${px.integrationName} / ${node.node}: root filesystem is ${pct(rootP)} full`,
|
||||
});
|
||||
}
|
||||
for (const st of node.storages) {
|
||||
if (!st.active) continue;
|
||||
const p = usage(st.usedBytes, st.totalBytes);
|
||||
if (p === null || p < limit) continue;
|
||||
if (st.shared) {
|
||||
// A shared storage is listed by every node — report it once, not once per node.
|
||||
if (seenShared.has(st.id)) continue;
|
||||
seenShared.add(st.id);
|
||||
out.push({
|
||||
key: `disk:proxmox:${px.integrationId}:storage:${st.id}`,
|
||||
source: `proxmox:${px.integrationId}:shared`,
|
||||
message: `${px.integrationName}: shared storage "${st.id}" is ${pct(p)} full (${formatBytes(st.usedBytes ?? 0)} of ${formatBytes(st.totalBytes ?? 0)})`,
|
||||
});
|
||||
} else {
|
||||
out.push({
|
||||
key: `disk:proxmox:${px.integrationId}:${node.node}:storage:${st.id}`,
|
||||
source: `proxmox:${px.integrationId}:${node.node}`,
|
||||
message: `${px.integrationName} / ${node.node}: storage "${st.id}" is ${pct(p)} full (${formatBytes(st.usedBytes ?? 0)} of ${formatBytes(st.totalBytes ?? 0)})`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const syn of snapshot.synology) {
|
||||
const src = `synology:${syn.integrationId}`;
|
||||
for (const v of syn.storage.volumes) {
|
||||
if (v.status && v.status.toLowerCase() !== "normal") {
|
||||
out.push({
|
||||
key: `synology-volume:${syn.integrationId}:${v.id}`,
|
||||
source: src,
|
||||
message: `${syn.integrationName}: volume ${v.id} status is "${v.status}"`,
|
||||
});
|
||||
}
|
||||
const p = usage(v.sizeUsed, v.sizeTotal);
|
||||
if (p !== null && p >= limit) {
|
||||
out.push({
|
||||
key: `disk:synology:${syn.integrationId}:${v.id}`,
|
||||
source: src,
|
||||
message: `${syn.integrationName}: volume ${v.id} is ${pct(p)} full (${formatBytes(v.sizeUsed ?? 0)} of ${formatBytes(v.sizeTotal ?? 0)})`,
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const d of syn.storage.disks) {
|
||||
const problems: string[] = [];
|
||||
if (d.status && d.status.toLowerCase() !== "normal") problems.push(`status "${d.status}"`);
|
||||
if (/warn|crit|fail|danger|bad/i.test(d.smartStatus ?? "")) problems.push(`SMART "${d.smartStatus}"`);
|
||||
if (d.exceedBadSectorThreshold) problems.push("bad sectors above threshold");
|
||||
if (d.belowRemainLifeThreshold) problems.push("remaining life below threshold");
|
||||
if (problems.length > 0) {
|
||||
out.push({
|
||||
key: `synology-disk:${syn.integrationId}:${d.id}`,
|
||||
source: src,
|
||||
message: `${syn.integrationName}: disk ${d.name || d.id} — ${problems.join(", ")}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── State diff (pure) ──────────────────────────────────────────────────────
|
||||
|
||||
export type ActiveState = Record<string, { message: string; source: string }>;
|
||||
|
||||
/** `held` is a set of hierarchical sources that couldn't be read this run; matches the source itself or anything beneath it. */
|
||||
function isHeld(source: string, held: Set<string>): boolean {
|
||||
for (const h of held) if (source === h || source.startsWith(`${h}:`)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares this run's conditions with what was already being reported.
|
||||
* A condition present now but not before is new; one present before but not
|
||||
* now is resolved — except when its source couldn't be read this run, in
|
||||
* which case it's carried over untouched. Without that, one failed poll would
|
||||
* report a recovery and then re-alert the same problem on the next poll.
|
||||
*/
|
||||
export function diffConditions(
|
||||
previous: ActiveState,
|
||||
current: HealthCondition[],
|
||||
held: Set<string> = new Set(),
|
||||
): { added: HealthCondition[]; resolved: { key: string; message: string }[]; next: ActiveState } {
|
||||
const next: ActiveState = {};
|
||||
for (const c of current) next[c.key] = { message: c.message, source: c.source };
|
||||
|
||||
const added = current.filter((c) => !(c.key in previous));
|
||||
const resolved: { key: string; message: string }[] = [];
|
||||
for (const [key, entry] of Object.entries(previous)) {
|
||||
if (key in next) continue;
|
||||
if (isHeld(entry.source, held)) {
|
||||
next[key] = entry;
|
||||
} else {
|
||||
resolved.push({ key, message: entry.message });
|
||||
}
|
||||
}
|
||||
return { added, resolved, next };
|
||||
}
|
||||
|
||||
// ─── Collection (I/O) ───────────────────────────────────────────────────────
|
||||
|
||||
async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; held: Set<string> }> {
|
||||
const held = new Set<string>();
|
||||
const snapshot: HealthSnapshot = { servers: [], proxmox: [], synology: [] };
|
||||
|
||||
for (const s of await db.select().from(servers)) {
|
||||
let disks: ServerSnapshot["disks"] = [];
|
||||
try {
|
||||
disks = s.disks ? JSON.parse(s.disks) : [];
|
||||
} catch {
|
||||
// a malformed report shouldn't stop every other server being checked
|
||||
}
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks });
|
||||
}
|
||||
|
||||
const rows = await db
|
||||
.select({ id: integrations.id, name: integrations.name, type: integrations.type })
|
||||
.from(integrations)
|
||||
.where(eq(integrations.enabled, true));
|
||||
|
||||
for (const row of rows) {
|
||||
if (row.type !== "proxmox" && row.type !== "synology") continue;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(row.id);
|
||||
if (!loaded) continue;
|
||||
if (row.type === "proxmox") {
|
||||
const nodes = await createProxmoxAdapter(loaded.config as any).listNodeStats();
|
||||
snapshot.proxmox.push({ integrationId: row.id, integrationName: row.name, nodes });
|
||||
for (const n of nodes) if (n.error) held.add(`proxmox:${row.id}:${n.node}`);
|
||||
if (nodes.length > 0 && nodes.every((n) => n.error)) held.add(`proxmox:${row.id}`);
|
||||
} else {
|
||||
const storage = await createSynologyAdapter(loaded.config as any).getStorageInfo();
|
||||
snapshot.synology.push({ integrationId: row.id, integrationName: row.name, storage });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[health] couldn't read ${row.type} integration ${row.id}:`, err instanceof Error ? err.message : err);
|
||||
held.add(`${row.type}:${row.id}`);
|
||||
}
|
||||
}
|
||||
return { snapshot, held };
|
||||
}
|
||||
|
||||
// ─── The scheduled pass ─────────────────────────────────────────────────────
|
||||
|
||||
async function loadState(): Promise<ActiveState> {
|
||||
const raw = await getInternalFlag(STATE_FLAG);
|
||||
if (!raw) return {};
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function runHealthCheck(now: number = Date.now()): Promise<{ added: number; resolved: number }> {
|
||||
const { healthChecks } = await getSettings();
|
||||
const { snapshot, held } = await collectSnapshot();
|
||||
|
||||
const inGrace = now - PROCESS_START < STARTUP_GRACE_MS;
|
||||
if (inGrace) held.add("server");
|
||||
|
||||
const current = evaluateHealth(snapshot, healthChecks, now, { skipServers: inGrace });
|
||||
const { added, resolved, next } = diffConditions(await loadState(), current, held);
|
||||
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
|
||||
|
||||
// State is tracked even when the alert toggle is off (the notify functions check it),
|
||||
// so turning alerts back on doesn't dump every long-standing condition at once.
|
||||
if (added.length > 0) await notifyHealthIssues(added);
|
||||
if (resolved.length > 0) await notifyHealthRecovered(resolved);
|
||||
return { added: added.length, resolved: resolved.length };
|
||||
}
|
||||
Reference in new issue
Block a user