/** * Alert rules of your own: "CPU above 90% for 15 minutes", "memory above 85%", "a disk is warmer than 55 °C". Each is a limit on one figure of the servers it covers (all of * them, one, or those with a tag), with the severity it should have. They are judged on the same pass as the built-in checks — what they find becomes ordinary health * conditions (`rule::server:[:]`) — so they are announced, recovered, held back while flapping, silenced by maintenance and routed like the rest. * * Load, memory and disk use are judged from the history the agents' reports build (a sample every ten to fifteen minutes): a rule with a time says the figure must have been * past its limit on every sample for that long. A disk's temperature is only known as of its last report, so it is judged right now. A server that is not reporting is not * judged — its own offline alert says that. */ import { and, eq, gte } from "drizzle-orm"; import { db } from "../db/client.js"; import { alertRules, serverMetrics, servers, type AlertRuleMetric } from "../db/schema.js"; import { parseHostHealth } from "./hostHealth.js"; import { InvalidTagError, normalizeTag, parseTags } from "./serverTags.js"; export type RuleRow = typeof alertRules.$inferSelect; export const MAX_FOR_MINUTES = 24 * 60; export const MAX_RULES = 100; /** A server whose newest sample is older than this is not judged: the figure is no longer "now". */ export const STALE_AFTER_MS = 45 * 60_000; export const METRIC_WORDS: Record = { cpu: { label: "CPU load", unit: "%" }, mem: { label: "Memory in use", unit: "%" }, disk: { label: "Disk use", unit: "%" }, disk_temp: { label: "Disk temperature", unit: " °C" }, }; export interface RuleFinding { ruleId: number; ruleName: string; serverId: number; serverName: string; /** The disk (mount point, or device for a temperature) it is about, if any. */ detail: string | null; value: number; severity: "warning" | "critical"; /** When this run of samples past the limit began (epoch ms); null when it is judged right now. */ since: number | null; message: string; } export interface RuleServer { id: number; name: string; tags: string[]; /** What the last report said about each disk's temperature. */ smart: { device: string; temperatureC: number | null }[]; } export type RuleSamples = Map; export const sampleKey = (serverId: number, metric: string) => `${serverId}|${metric}`; const durationWords = (minutes: number) => (minutes < 60 ? `${minutes} min` : minutes % 60 === 0 ? `${minutes / 60} h` : `${Math.floor(minutes / 60)} h ${minutes % 60} min`); const breaches = (rule: Pick, v: number) => (rule.operator === "above" ? v > rule.threshold : v < rule.threshold); export function ruleCovers(rule: Pick, server: Pick): boolean { if (rule.serverId !== null) return rule.serverId === server.id; if (rule.tag) return server.tags.some((t) => t.toLowerCase() === rule.tag!.toLowerCase()); return true; } /** * Pure: how long the newest samples have been past the limit without a break. Returns the figure now and when the unbroken run began, or null when the newest sample is * not past the limit (or is too old to say). */ export function currentRun(rule: Pick, samples: { t: number; v: number }[], now: number): { value: number; since: number; latest: number } | null { if (samples.length === 0) return null; const sorted = [...samples].sort((a, b) => a.t - b.t); const newest = sorted[sorted.length - 1]; if (now - newest.t > STALE_AFTER_MS || !breaches(rule, newest.v)) return null; let since = newest.t; for (let i = sorted.length - 2; i >= 0; i--) { if (!breaches(rule, sorted[i].v)) break; since = sorted[i].t; } return { value: newest.v, since, latest: newest.t }; } /** Pure: what the rules find on these servers, from their samples (and, for temperatures, the last report). */ export function evaluateRules(rules: RuleRow[], serversIn: RuleServer[], samples: RuleSamples, now: number): RuleFinding[] { const out: RuleFinding[] = []; for (const rule of rules) { if (!rule.enabled) continue; const word = METRIC_WORDS[rule.metric]; const limit = `${rule.operator} ${rule.threshold}${word.unit}`; for (const s of serversIn) { if (!ruleCovers(rule, s)) continue; if (rule.metric === "disk_temp") { for (const d of s.smart) { if (d.temperatureC === null || !breaches(rule, d.temperatureC)) continue; out.push({ ruleId: rule.id, ruleName: rule.name, serverId: s.id, serverName: s.name, detail: d.device, value: d.temperatureC, severity: rule.severity, since: null, message: `${s.name}: ${d.device} is ${Math.round(d.temperatureC)}${word.unit}, ${limit} [${rule.name}]` }); } continue; } const metrics = rule.metric === "disk" ? [...samples.keys()].filter((k) => k.startsWith(`${s.id}|disk:`) && (rule.mount === null || k === sampleKey(s.id, `disk:${rule.mount}`))) : [sampleKey(s.id, rule.metric)]; for (const k of metrics) { const run = currentRun(rule, samples.get(k) ?? [], now); if (!run) continue; // The figure must have been past the limit for the whole time: the run began that long before the newest sample. const heldFor = Math.floor((run.latest - run.since) / 60_000); if (rule.forMinutes > 0 && heldFor < rule.forMinutes) continue; const detail = rule.metric === "disk" ? k.slice(`${s.id}|disk:`.length) : null; const what = rule.metric === "disk" ? `${s.name}: ${detail} use` : `${s.name}: ${word.label.toLowerCase()}`; out.push({ ruleId: rule.id, ruleName: rule.name, serverId: s.id, serverName: s.name, detail, value: run.value, severity: rule.severity, since: run.since, message: `${what} is ${Math.round(run.value)}${word.unit}, ${limit}${rule.forMinutes > 0 ? ` for ${durationWords(rule.forMinutes)}` : ""} [${rule.name}]`, }); } } } return out; } /** The condition key a finding has: the same finding always has the same key, so it is announced once. */ export const findingKey = (f: Pick) => `rule:${f.ruleId}:server:${f.serverId}${f.detail ? `:${f.detail}` : ""}`; export async function listRules(): Promise { return (await db.select().from(alertRules)).sort((a, b) => a.name.localeCompare(b.name) || a.id - b.id); } /** What the given rules find right now, from the stored history — on one server only when `onlyServerId` is given, which reads only that server's history. Reads nothing when no rule is on. */ export async function computeRuleFindings(rules: RuleRow[], now: number = Date.now(), onlyServerId?: number): Promise { const on = rules.filter((r) => r.enabled); if (on.length === 0) return []; const everything = db.select({ id: servers.id, name: servers.name, tags: servers.tags, hostHealth: servers.hostHealth }).from(servers); const rows = await (onlyServerId === undefined ? everything : everything.where(eq(servers.id, onlyServerId))); const serverList: RuleServer[] = rows.map((r) => ({ id: r.id, name: r.name, tags: parseTags(r.tags), smart: (parseHostHealth(r.hostHealth)?.smart ?? []).map((d) => ({ device: d.device, temperatureC: d.temperatureC })), })); const samples: RuleSamples = new Map(); if (on.some((r) => r.metric !== "disk_temp")) { const longest = Math.max(...on.filter((r) => r.metric !== "disk_temp").map((r) => r.forMinutes)); const since = new Date(now - (longest * 60_000 + 3 * 3600_000)).toISOString(); const recent = gte(serverMetrics.recordedAt, since); for (const r of await db.select().from(serverMetrics).where(onlyServerId === undefined ? recent : and(recent, eq(serverMetrics.serverId, onlyServerId)))) { if (r.metric !== "cpu" && r.metric !== "mem" && !r.metric.startsWith("disk:")) continue; const k = sampleKey(r.serverId, r.metric); const list = samples.get(k) ?? []; list.push({ t: Date.parse(r.recordedAt), v: r.value }); samples.set(k, list); } } return evaluateRules(on, serverList, samples, now); } // ─── Making and changing rules ───────────────────────────────────────────── export class RuleError extends Error { constructor( readonly status: number, readonly code: string, message: string, ) { super(message); } } export interface RuleInput { name: string; enabled?: boolean; metric: AlertRuleMetric; mount?: string | null; operator?: "above" | "below"; threshold: number; forMinutes?: number; severity?: "warning" | "critical"; serverId?: number | null; tag?: string | null; } /** Checks and tidies what was asked for; says plainly what is wrong with it. */ export async function checkedRule(input: RuleInput): Promise> { const name = input.name.trim(); if (name === "" || name.length > 80) throw new RuleError(400, "invalid_name", "Give the rule a name, up to 80 characters."); const operator = input.operator ?? "above"; if (!Number.isFinite(input.threshold)) throw new RuleError(400, "invalid_threshold", "The limit has to be a number."); const percent = input.metric !== "disk_temp"; if (percent && (input.threshold < 0 || input.threshold > 100)) throw new RuleError(400, "invalid_threshold", "A percentage limit is between 0 and 100."); if (!percent && (input.threshold < 0 || input.threshold > 150)) throw new RuleError(400, "invalid_threshold", "A temperature limit is between 0 and 150 °C."); const forMinutes = percent ? Math.round(input.forMinutes ?? 0) : 0; if (!Number.isInteger(forMinutes) || forMinutes < 0 || forMinutes > MAX_FOR_MINUTES) throw new RuleError(400, "invalid_for", "How long it has to hold is from 0 to 1440 minutes (a day)."); const mount = input.metric === "disk" ? input.mount?.trim() || null : null; if (mount && mount.length > 200) throw new RuleError(400, "invalid_mount", "That disk name is too long."); let tag: string | null = null; if (input.tag?.trim()) { try { tag = normalizeTag(input.tag); } catch (err) { if (err instanceof InvalidTagError) throw new RuleError(400, "invalid_tag", err.message); throw err; } } const serverId = input.serverId ?? null; if (serverId !== null && tag) throw new RuleError(400, "invalid_scope", "A rule is for one server, or for servers with a tag, not both."); if (serverId !== null) { const [row] = await db.select({ id: servers.id }).from(servers).where(eq(servers.id, serverId)).limit(1); if (!row) throw new RuleError(400, "unknown_server", "That server is not there."); } return { name, enabled: input.enabled ?? true, metric: input.metric, mount, operator, threshold: input.threshold, forMinutes, severity: input.severity ?? "warning", serverId, tag }; }