Bulk snooze: Snooze... and Wake on the ticked alerts, with undo that puts back each alert's earlier snooze exactly. Alert rules are judged for a server right after its agent reports, instead of at the next 15-minute pass; only that server's rule findings change, and the check is serialized with the pass so neither overwrites the other's state. Photos on server and device notes: up to 12 each, shrunk in the browser, accepted by their own bytes (JPEG/PNG/GIF/WebP, 4 MB), kept in the database and removed with the server or device. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
214 lines
11 KiB
TypeScript
214 lines
11 KiB
TypeScript
/**
|
|
* Alert rules of your own: "CPU above 90% for 15 minutes", "memory above 85%", "a disk is warmer than 55 °C". Each is a limit on one figure of the servers it covers (all of
|
|
* them, one, or those with a tag), with the severity it should have. They are judged on the same pass as the built-in checks — what they find becomes ordinary health
|
|
* conditions (`rule:<id>:server:<server>[:<disk>]`) — so they are announced, recovered, held back while flapping, silenced by maintenance and routed like the rest.
|
|
*
|
|
* Load, memory and disk use are judged from the history the agents' reports build (a sample every ten to fifteen minutes): a rule with a time says the figure must have been
|
|
* past its limit on every sample for that long. A disk's temperature is only known as of its last report, so it is judged right now. A server that is not reporting is not
|
|
* judged — its own offline alert says that.
|
|
*/
|
|
import { and, eq, gte } from "drizzle-orm";
|
|
import { db } from "../db/client.js";
|
|
import { alertRules, serverMetrics, servers, type AlertRuleMetric } from "../db/schema.js";
|
|
import { parseHostHealth } from "./hostHealth.js";
|
|
import { InvalidTagError, normalizeTag, parseTags } from "./serverTags.js";
|
|
|
|
export type RuleRow = typeof alertRules.$inferSelect;
|
|
|
|
export const MAX_FOR_MINUTES = 24 * 60;
|
|
export const MAX_RULES = 100;
|
|
/** A server whose newest sample is older than this is not judged: the figure is no longer "now". */
|
|
export const STALE_AFTER_MS = 45 * 60_000;
|
|
|
|
export const METRIC_WORDS: Record<AlertRuleMetric, { label: string; unit: string }> = {
|
|
cpu: { label: "CPU load", unit: "%" },
|
|
mem: { label: "Memory in use", unit: "%" },
|
|
disk: { label: "Disk use", unit: "%" },
|
|
disk_temp: { label: "Disk temperature", unit: " °C" },
|
|
};
|
|
|
|
export interface RuleFinding {
|
|
ruleId: number;
|
|
ruleName: string;
|
|
serverId: number;
|
|
serverName: string;
|
|
/** The disk (mount point, or device for a temperature) it is about, if any. */
|
|
detail: string | null;
|
|
value: number;
|
|
severity: "warning" | "critical";
|
|
/** When this run of samples past the limit began (epoch ms); null when it is judged right now. */
|
|
since: number | null;
|
|
message: string;
|
|
}
|
|
|
|
export interface RuleServer {
|
|
id: number;
|
|
name: string;
|
|
tags: string[];
|
|
/** What the last report said about each disk's temperature. */
|
|
smart: { device: string; temperatureC: number | null }[];
|
|
}
|
|
|
|
export type RuleSamples = Map<string, { t: number; v: number }[]>;
|
|
export const sampleKey = (serverId: number, metric: string) => `${serverId}|${metric}`;
|
|
|
|
const durationWords = (minutes: number) => (minutes < 60 ? `${minutes} min` : minutes % 60 === 0 ? `${minutes / 60} h` : `${Math.floor(minutes / 60)} h ${minutes % 60} min`);
|
|
|
|
const breaches = (rule: Pick<RuleRow, "operator" | "threshold">, v: number) => (rule.operator === "above" ? v > rule.threshold : v < rule.threshold);
|
|
|
|
export function ruleCovers(rule: Pick<RuleRow, "serverId" | "tag">, server: Pick<RuleServer, "id" | "tags">): boolean {
|
|
if (rule.serverId !== null) return rule.serverId === server.id;
|
|
if (rule.tag) return server.tags.some((t) => t.toLowerCase() === rule.tag!.toLowerCase());
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Pure: how long the newest samples have been past the limit without a break. Returns the figure now and when the unbroken run began, or null when the newest sample is
|
|
* not past the limit (or is too old to say).
|
|
*/
|
|
export function currentRun(rule: Pick<RuleRow, "operator" | "threshold">, samples: { t: number; v: number }[], now: number): { value: number; since: number; latest: number } | null {
|
|
if (samples.length === 0) return null;
|
|
const sorted = [...samples].sort((a, b) => a.t - b.t);
|
|
const newest = sorted[sorted.length - 1];
|
|
if (now - newest.t > STALE_AFTER_MS || !breaches(rule, newest.v)) return null;
|
|
let since = newest.t;
|
|
for (let i = sorted.length - 2; i >= 0; i--) {
|
|
if (!breaches(rule, sorted[i].v)) break;
|
|
since = sorted[i].t;
|
|
}
|
|
return { value: newest.v, since, latest: newest.t };
|
|
}
|
|
|
|
/** Pure: what the rules find on these servers, from their samples (and, for temperatures, the last report). */
|
|
export function evaluateRules(rules: RuleRow[], serversIn: RuleServer[], samples: RuleSamples, now: number): RuleFinding[] {
|
|
const out: RuleFinding[] = [];
|
|
for (const rule of rules) {
|
|
if (!rule.enabled) continue;
|
|
const word = METRIC_WORDS[rule.metric];
|
|
const limit = `${rule.operator} ${rule.threshold}${word.unit}`;
|
|
for (const s of serversIn) {
|
|
if (!ruleCovers(rule, s)) continue;
|
|
if (rule.metric === "disk_temp") {
|
|
for (const d of s.smart) {
|
|
if (d.temperatureC === null || !breaches(rule, d.temperatureC)) continue;
|
|
out.push({ ruleId: rule.id, ruleName: rule.name, serverId: s.id, serverName: s.name, detail: d.device, value: d.temperatureC, severity: rule.severity, since: null, message: `${s.name}: ${d.device} is ${Math.round(d.temperatureC)}${word.unit}, ${limit} [${rule.name}]` });
|
|
}
|
|
continue;
|
|
}
|
|
const metrics = rule.metric === "disk" ? [...samples.keys()].filter((k) => k.startsWith(`${s.id}|disk:`) && (rule.mount === null || k === sampleKey(s.id, `disk:${rule.mount}`))) : [sampleKey(s.id, rule.metric)];
|
|
for (const k of metrics) {
|
|
const run = currentRun(rule, samples.get(k) ?? [], now);
|
|
if (!run) continue;
|
|
// The figure must have been past the limit for the whole time: the run began that long before the newest sample.
|
|
const heldFor = Math.floor((run.latest - run.since) / 60_000);
|
|
if (rule.forMinutes > 0 && heldFor < rule.forMinutes) continue;
|
|
const detail = rule.metric === "disk" ? k.slice(`${s.id}|disk:`.length) : null;
|
|
const what = rule.metric === "disk" ? `${s.name}: ${detail} use` : `${s.name}: ${word.label.toLowerCase()}`;
|
|
out.push({
|
|
ruleId: rule.id,
|
|
ruleName: rule.name,
|
|
serverId: s.id,
|
|
serverName: s.name,
|
|
detail,
|
|
value: run.value,
|
|
severity: rule.severity,
|
|
since: run.since,
|
|
message: `${what} is ${Math.round(run.value)}${word.unit}, ${limit}${rule.forMinutes > 0 ? ` for ${durationWords(rule.forMinutes)}` : ""} [${rule.name}]`,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/** The condition key a finding has: the same finding always has the same key, so it is announced once. */
|
|
export const findingKey = (f: Pick<RuleFinding, "ruleId" | "serverId" | "detail">) => `rule:${f.ruleId}:server:${f.serverId}${f.detail ? `:${f.detail}` : ""}`;
|
|
|
|
export async function listRules(): Promise<RuleRow[]> {
|
|
return (await db.select().from(alertRules)).sort((a, b) => a.name.localeCompare(b.name) || a.id - b.id);
|
|
}
|
|
|
|
/** What the given rules find right now, from the stored history — on one server only when `onlyServerId` is given, which reads only that server's history. Reads nothing when no rule is on. */
|
|
export async function computeRuleFindings(rules: RuleRow[], now: number = Date.now(), onlyServerId?: number): Promise<RuleFinding[]> {
|
|
const on = rules.filter((r) => r.enabled);
|
|
if (on.length === 0) return [];
|
|
const everything = db.select({ id: servers.id, name: servers.name, tags: servers.tags, hostHealth: servers.hostHealth }).from(servers);
|
|
const rows = await (onlyServerId === undefined ? everything : everything.where(eq(servers.id, onlyServerId)));
|
|
const serverList: RuleServer[] = rows.map((r) => ({
|
|
id: r.id,
|
|
name: r.name,
|
|
tags: parseTags(r.tags),
|
|
smart: (parseHostHealth(r.hostHealth)?.smart ?? []).map((d) => ({ device: d.device, temperatureC: d.temperatureC })),
|
|
}));
|
|
const samples: RuleSamples = new Map();
|
|
if (on.some((r) => r.metric !== "disk_temp")) {
|
|
const longest = Math.max(...on.filter((r) => r.metric !== "disk_temp").map((r) => r.forMinutes));
|
|
const since = new Date(now - (longest * 60_000 + 3 * 3600_000)).toISOString();
|
|
const recent = gte(serverMetrics.recordedAt, since);
|
|
for (const r of await db.select().from(serverMetrics).where(onlyServerId === undefined ? recent : and(recent, eq(serverMetrics.serverId, onlyServerId)))) {
|
|
if (r.metric !== "cpu" && r.metric !== "mem" && !r.metric.startsWith("disk:")) continue;
|
|
const k = sampleKey(r.serverId, r.metric);
|
|
const list = samples.get(k) ?? [];
|
|
list.push({ t: Date.parse(r.recordedAt), v: r.value });
|
|
samples.set(k, list);
|
|
}
|
|
}
|
|
return evaluateRules(on, serverList, samples, now);
|
|
}
|
|
|
|
// ─── Making and changing rules ─────────────────────────────────────────────
|
|
|
|
export class RuleError extends Error {
|
|
constructor(
|
|
readonly status: number,
|
|
readonly code: string,
|
|
message: string,
|
|
) {
|
|
super(message);
|
|
}
|
|
}
|
|
|
|
export interface RuleInput {
|
|
name: string;
|
|
enabled?: boolean;
|
|
metric: AlertRuleMetric;
|
|
mount?: string | null;
|
|
operator?: "above" | "below";
|
|
threshold: number;
|
|
forMinutes?: number;
|
|
severity?: "warning" | "critical";
|
|
serverId?: number | null;
|
|
tag?: string | null;
|
|
}
|
|
|
|
/** Checks and tidies what was asked for; says plainly what is wrong with it. */
|
|
export async function checkedRule(input: RuleInput): Promise<Omit<RuleRow, "id" | "createdAt" | "createdBy">> {
|
|
const name = input.name.trim();
|
|
if (name === "" || name.length > 80) throw new RuleError(400, "invalid_name", "Give the rule a name, up to 80 characters.");
|
|
const operator = input.operator ?? "above";
|
|
if (!Number.isFinite(input.threshold)) throw new RuleError(400, "invalid_threshold", "The limit has to be a number.");
|
|
const percent = input.metric !== "disk_temp";
|
|
if (percent && (input.threshold < 0 || input.threshold > 100)) throw new RuleError(400, "invalid_threshold", "A percentage limit is between 0 and 100.");
|
|
if (!percent && (input.threshold < 0 || input.threshold > 150)) throw new RuleError(400, "invalid_threshold", "A temperature limit is between 0 and 150 °C.");
|
|
const forMinutes = percent ? Math.round(input.forMinutes ?? 0) : 0;
|
|
if (!Number.isInteger(forMinutes) || forMinutes < 0 || forMinutes > MAX_FOR_MINUTES) throw new RuleError(400, "invalid_for", "How long it has to hold is from 0 to 1440 minutes (a day).");
|
|
const mount = input.metric === "disk" ? input.mount?.trim() || null : null;
|
|
if (mount && mount.length > 200) throw new RuleError(400, "invalid_mount", "That disk name is too long.");
|
|
let tag: string | null = null;
|
|
if (input.tag?.trim()) {
|
|
try {
|
|
tag = normalizeTag(input.tag);
|
|
} catch (err) {
|
|
if (err instanceof InvalidTagError) throw new RuleError(400, "invalid_tag", err.message);
|
|
throw err;
|
|
}
|
|
}
|
|
const serverId = input.serverId ?? null;
|
|
if (serverId !== null && tag) throw new RuleError(400, "invalid_scope", "A rule is for one server, or for servers with a tag, not both.");
|
|
if (serverId !== null) {
|
|
const [row] = await db.select({ id: servers.id }).from(servers).where(eq(servers.id, serverId)).limit(1);
|
|
if (!row) throw new RuleError(400, "unknown_server", "That server is not there.");
|
|
}
|
|
return { name, enabled: input.enabled ?? true, metric: input.metric, mount, operator, threshold: input.threshold, forMinutes, severity: input.severity ?? "warning", serverId, tag };
|
|
}
|