Alert rules of your own, storage forecasts for Proxmox and Synology, and snooze
- Monitoring -> Alert rules: CPU, memory, disk use and disk temperature limits with a time, severity and scope (all, one server, a tag); try before saving; findings are ordinary health conditions - Proxmox storages and Synology volumes are written down on each health pass (hourly, 400 days) and forecast like server disks: badge and history chart on the pages, alert, digest line, /metrics gauge - Snooze an alert until a time; held back like maintenance, messages held, announced once again if still there when it ends - Docs updated Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
788b3e073c
commit
24ae8db380
38 files changed
+7847
-8
No files matched your search
@@ -0,0 +1,50 @@
|
||||
CREATE TABLE `alert_rules` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`name` text NOT NULL,
|
||||
`enabled` integer DEFAULT true NOT NULL,
|
||||
`metric` text NOT NULL,
|
||||
`mount` text,
|
||||
`operator` text DEFAULT 'above' NOT NULL,
|
||||
`threshold` real NOT NULL,
|
||||
`for_minutes` integer DEFAULT 0 NOT NULL,
|
||||
`severity` text DEFAULT 'warning' NOT NULL,
|
||||
`server_id` integer,
|
||||
`tag` text,
|
||||
`created_by` text,
|
||||
`created_at` text NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE `alert_snoozes` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`alert_id` text NOT NULL,
|
||||
`condition_key` text NOT NULL,
|
||||
`message` text NOT NULL,
|
||||
`until` text NOT NULL,
|
||||
`snoozed_by` text NOT NULL,
|
||||
`snoozed_at` text NOT NULL,
|
||||
`last_seen_at` text NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX `alert_snoozes_alert_idx` ON `alert_snoozes` (`alert_id`);--> statement-breakpoint
|
||||
CREATE TABLE `storage_samples` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`series_id` integer NOT NULL,
|
||||
`at` text NOT NULL,
|
||||
`value` real NOT NULL,
|
||||
FOREIGN KEY (`series_id`) REFERENCES `storage_series`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `storage_samples_series_idx` ON `storage_samples` (`series_id`,`at`);--> statement-breakpoint
|
||||
CREATE TABLE `storage_series` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`integration_id` integer NOT NULL,
|
||||
`key` text NOT NULL,
|
||||
`label` text NOT NULL,
|
||||
`latest_percent` real NOT NULL,
|
||||
`used_bytes` integer,
|
||||
`total_bytes` integer,
|
||||
`updated_at` text NOT NULL,
|
||||
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX `storage_series_key_idx` ON `storage_series` (`integration_id`,`key`);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -358,6 +358,13 @@
|
||||
"when": 1792523816802,
|
||||
"tag": "0050_useful_bulldozer",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 51,
|
||||
"version": "6",
|
||||
"when": 1792531006435,
|
||||
"tag": "0051_next_nitro",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -50,6 +50,8 @@ import { myWorkRouter } from "./routes/myWork.js";
|
||||
import { apiTokensRouter } from "./routes/apiTokens.js";
|
||||
import { notesRouter, todosRouter } from "./routes/todos.js";
|
||||
import { npmRouter } from "./routes/npm.js";
|
||||
import { alertRulesRouter } from "./routes/alertRules.js";
|
||||
import { storageRouter } from "./routes/storage.js";
|
||||
import { homeAssistantRouter } from "./routes/homeAssistant.js";
|
||||
import { publicApiRouter } from "./routes/publicApi.js";
|
||||
import { generatorRouter } from "./routes/generator.js";
|
||||
@@ -174,6 +176,8 @@ export function createApp() {
|
||||
app.use("/api/network", networkRouter);
|
||||
app.use("/api/my-work", myWorkRouter);
|
||||
app.use("/api/npm", npmRouter);
|
||||
app.use("/api/alert-rules", alertRulesRouter);
|
||||
app.use("/api/storage", storageRouter);
|
||||
app.use("/api/home-assistant", homeAssistantRouter);
|
||||
app.use("/api/todos", todosRouter);
|
||||
app.use("/api/notes", notesRouter);
|
||||
|
||||
@@ -1224,3 +1224,73 @@ export const eventWebhooks = sqliteTable("event_webhooks", {
|
||||
lastAttemptAt: text("last_attempt_at"), // ISO
|
||||
lastResult: text("last_result"), // "200", or what went wrong
|
||||
});
|
||||
|
||||
// ─── Your own alert rules, storage history and snoozes ──────────────────────
|
||||
|
||||
export const alertRuleMetrics = ["cpu", "mem", "disk", "disk_temp"] as const;
|
||||
export type AlertRuleMetric = (typeof alertRuleMetrics)[number];
|
||||
|
||||
// "CPU above 90% for 15 minutes", "a disk is warmer than 55 °C": a limit of the administrator's own, judged for every server (or one, or those with a tag) on the
|
||||
// same pass as the built-in checks, so it alerts, recovers, flaps, is silenced by maintenance and is routed like them — see services/alertRules.ts.
|
||||
export const alertRules = sqliteTable("alert_rules", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
name: text("name").notNull(),
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
metric: text("metric").$type<AlertRuleMetric>().notNull(),
|
||||
mount: text("mount"), // for "disk": one mount point; null = every one
|
||||
operator: text("operator").$type<"above" | "below">().notNull().default("above"),
|
||||
threshold: real("threshold").notNull(), // percent, or °C for disk_temp
|
||||
forMinutes: integer("for_minutes").notNull().default(0), // how long it must have held (0 = right now); not used for disk_temp, which is only known as of the last report
|
||||
severity: text("severity").$type<"warning" | "critical">().notNull().default("warning"),
|
||||
serverId: integer("server_id"), // one server; null = see tag. A plain reference: a rule for a server that is gone judges nothing.
|
||||
tag: text("tag"), // servers with this tag; null with a null server = every server
|
||||
createdBy: text("created_by"),
|
||||
createdAt: text("created_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// What each pool of an integration (a Proxmox storage, a Synology volume) looked like, one reading an hour, so it can be forecast the way an agent server's disk is.
|
||||
export const storageSeries = sqliteTable(
|
||||
"storage_series",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
integrationId: integer("integration_id")
|
||||
.notNull()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
key: text("key").notNull(), // "proxmox:<node>:storage:<id>", "proxmox:shared:<id>", "proxmox:<node>:rootfs", "synology:<volume>"
|
||||
label: text("label").notNull(), // "Storage local-lvm on pve1"
|
||||
latestPercent: real("latest_percent").notNull(),
|
||||
usedBytes: integer("used_bytes"),
|
||||
totalBytes: integer("total_bytes"),
|
||||
updatedAt: text("updated_at").notNull(), // ISO; the last reading
|
||||
},
|
||||
(t) => [uniqueIndex("storage_series_key_idx").on(t.integrationId, t.key)],
|
||||
);
|
||||
|
||||
export const storageSamples = sqliteTable(
|
||||
"storage_samples",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
seriesId: integer("series_id")
|
||||
.notNull()
|
||||
.references(() => storageSeries.id, { onDelete: "cascade" }),
|
||||
at: text("at").notNull(), // ISO
|
||||
value: real("value").notNull(), // percent used
|
||||
},
|
||||
(t) => [index("storage_samples_series_idx").on(t.seriesId, t.at)],
|
||||
);
|
||||
|
||||
// "Remind me tomorrow at 09:00": an alert hidden until a time, and announced again then if it is still there — see services/alertSnoozes.ts. Keyed like an acknowledgement.
|
||||
export const alertSnoozes = sqliteTable(
|
||||
"alert_snoozes",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
alertId: text("alert_id").notNull(), // the alert's id on the Alerts page
|
||||
conditionKey: text("condition_key").notNull(), // the key behind it, which the notifications carry
|
||||
message: text("message").notNull(), // what it said, for the reminder
|
||||
until: text("until").notNull(), // ISO
|
||||
snoozedBy: text("snoozed_by").notNull(),
|
||||
snoozedAt: text("snoozed_at").notNull(), // ISO
|
||||
lastSeenAt: text("last_seen_at").notNull(), // ISO; the last time the alert was there
|
||||
},
|
||||
(t) => [uniqueIndex("alert_snoozes_alert_idx").on(t.alertId)],
|
||||
);
|
||||
@@ -24,6 +24,7 @@ import { initEscalation } from "./services/escalation.js";
|
||||
import { initEndpointMonitor } from "./services/endpointMonitor.js";
|
||||
import { initPatchPlans } from "./services/patchPlans.js";
|
||||
import { initFlapMonitor } from "./services/flapMonitor.js";
|
||||
import { initSnoozeMonitor } from "./services/snoozeMonitor.js";
|
||||
import { initUnifiSync } from "./services/unifiSync.js";
|
||||
import { initHomeAssistantSync } from "./services/homeAssistantSync.js";
|
||||
import { backfillMetricRollups } from "./services/serverMetrics.js";
|
||||
@@ -71,6 +72,7 @@ initEscalation();
|
||||
initEndpointMonitor();
|
||||
initPatchPlans();
|
||||
initFlapMonitor();
|
||||
initSnoozeMonitor();
|
||||
initUnifiSync();
|
||||
initHomeAssistantSync();
|
||||
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { alertRules, alertRuleMetrics, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { checkedRule, computeRuleFindings, listRules, MAX_RULES, RuleError, type RuleRow } from "../services/alertRules.js";
|
||||
import { parseTags } from "../services/serverTags.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
/** Alert rules of your own: everyone signed in sees them and what they find now; administrators make, change and remove them. */
|
||||
export const alertRulesRouter = Router();
|
||||
alertRulesRouter.use(requireAuth);
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
const body = z.object({
|
||||
name: z.string(),
|
||||
enabled: z.boolean().optional(),
|
||||
metric: z.enum(alertRuleMetrics),
|
||||
mount: z.string().nullable().optional(),
|
||||
operator: z.enum(["above", "below"]).optional(),
|
||||
threshold: z.number(),
|
||||
forMinutes: z.number().optional(),
|
||||
severity: z.enum(["warning", "critical"]).optional(),
|
||||
serverId: z.number().int().positive().nullable().optional(),
|
||||
tag: z.string().nullable().optional(),
|
||||
});
|
||||
|
||||
const shape = (r: RuleRow, findings: Awaited<ReturnType<typeof computeRuleFindings>>, names: Map<number, string>) => ({
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
enabled: r.enabled,
|
||||
metric: r.metric,
|
||||
mount: r.mount,
|
||||
operator: r.operator,
|
||||
threshold: r.threshold,
|
||||
forMinutes: r.forMinutes,
|
||||
severity: r.severity,
|
||||
serverId: r.serverId,
|
||||
serverName: r.serverId !== null ? (names.get(r.serverId) ?? null) : null,
|
||||
tag: r.tag,
|
||||
createdBy: r.createdBy,
|
||||
createdAt: r.createdAt,
|
||||
firing: findings.filter((f) => f.ruleId === r.id).map((f) => ({ serverId: f.serverId, serverName: f.serverName, detail: f.detail, value: f.value, since: f.since === null ? null : new Date(f.since).toISOString() })),
|
||||
});
|
||||
|
||||
alertRulesRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const rules = await listRules();
|
||||
const findings = await computeRuleFindings(rules);
|
||||
const serverRows = await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers);
|
||||
const names = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
const tags = [...new Set(serverRows.flatMap((s) => parseTags(s.tags)))].sort((a, b) => a.localeCompare(b, "sv"));
|
||||
res.json({ rules: rules.map((r) => shape(r, findings, names)), servers: serverRows.map((s) => ({ id: s.id, name: s.name })).sort((a, b) => a.name.localeCompare(b.name)), tags });
|
||||
}));
|
||||
|
||||
// What a rule that has not been saved would find right now, so it can be tried before it is.
|
||||
alertRulesRouter.post("/preview", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = body.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Fill in what to watch and the limit." });
|
||||
try {
|
||||
const checked = await checkedRule(parsed.data);
|
||||
const findings = await computeRuleFindings([{ id: 0, createdAt: "", createdBy: null, ...checked, enabled: true }]);
|
||||
res.json({ findings: findings.map((f) => ({ serverId: f.serverId, serverName: f.serverName, detail: f.detail, value: f.value })) });
|
||||
} catch (err) {
|
||||
if (err instanceof RuleError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
alertRulesRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = body.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Fill in a name, what to watch and the limit." });
|
||||
try {
|
||||
if ((await listRules()).length >= MAX_RULES) throw new RuleError(400, "too_many", `That is as many rules as can be kept (${MAX_RULES}).`);
|
||||
const checked = await checkedRule(parsed.data);
|
||||
const actor = req.currentUser!;
|
||||
const [row] = await db.insert(alertRules).values({ ...checked, createdBy: actor.name ?? actor.email ?? actor.oidcSub, createdAt: new Date().toISOString() }).returning();
|
||||
await recordAudit({ actor, category: "alert_rule", action: "create", targetType: "alert_rule", targetId: row.id, detail: { name: row.name, metric: row.metric, operator: row.operator, threshold: row.threshold, forMinutes: row.forMinutes, severity: row.severity } });
|
||||
invalidateAlertsCache();
|
||||
res.status(201).json({ rule: shape(row, [], new Map()) });
|
||||
} catch (err) {
|
||||
if (err instanceof RuleError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
alertRulesRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = body.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Fill in a name, what to watch and the limit." });
|
||||
const [before] = await db.select().from(alertRules).where(eq(alertRules.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
try {
|
||||
const checked = await checkedRule(parsed.data);
|
||||
const [row] = await db.update(alertRules).set(checked).where(eq(alertRules.id, id)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "alert_rule", action: before.enabled !== row.enabled && before.name === row.name ? (row.enabled ? "enable" : "disable") : "update", targetType: "alert_rule", targetId: id, detail: { name: row.name, metric: row.metric, operator: row.operator, threshold: row.threshold, forMinutes: row.forMinutes, severity: row.severity } });
|
||||
invalidateAlertsCache();
|
||||
res.json({ rule: shape(row, [], new Map()) });
|
||||
} catch (err) {
|
||||
if (err instanceof RuleError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
alertRulesRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.select().from(alertRules).where(eq(alertRules.id, id)).limit(1);
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(alertRules).where(eq(alertRules.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "alert_rule", action: "delete", targetType: "alert_rule", targetId: id, detail: { name: row.name } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -1,7 +1,9 @@
|
||||
import { Router } from "express";
|
||||
import { z } from "zod";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { getAlerts } from "../services/alerts.js";
|
||||
import { getAlerts, invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { clearSnooze, setSnooze, SnoozeError } from "../services/alertSnoozes.js";
|
||||
import { keyOfAlertId } from "../services/flapGuard.js";
|
||||
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { openIncidentIds } from "../services/incidents.js";
|
||||
@@ -95,3 +97,39 @@ alertsRouter.delete("/ack", requireRole("operator"), asyncHandler(async (req, re
|
||||
await recordAudit({ actor: req.currentUser!, category: "alert", action: "unack", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: alert?.message ?? null } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
const snoozeSchema = z.object({
|
||||
alertId: z.string().min(1).max(600),
|
||||
/** When it should come back, as a date and time with its zone. */
|
||||
until: z.string().min(1).max(40),
|
||||
});
|
||||
|
||||
// "Remind me then": hide an alert until a time, and announce it again then if it is still there.
|
||||
alertsRouter.put("/snooze", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = snoozeSchema.safeParse(req.body);
|
||||
const until = parsed.success ? new Date(parsed.data.until) : null;
|
||||
if (!parsed.success || !until || Number.isNaN(until.getTime())) return res.status(400).json({ error: "invalid_body", message: "Say which alert, and when it should come back." });
|
||||
const alert = (await getAlerts(false)).alerts.find((a) => a.id === parsed.data.alertId);
|
||||
if (!alert) return res.status(404).json({ error: "not_found", message: "That alert isn't there any more." });
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
const snooze = await setSnooze({ alertId: alert.id, conditionKey: keyOfAlertId(alert.id, alert.category), message: alert.message, until, actor: actor.name ?? actor.email ?? actor.oidcSub });
|
||||
await recordAudit({ actor, category: "alert", action: "snooze", ...auditTargetOf(alert), detail: { alertId: alert.id, message: alert.message, until: snooze.until } });
|
||||
invalidateAlertsCache();
|
||||
res.json({ snooze });
|
||||
} catch (err) {
|
||||
if (err instanceof SnoozeError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
alertsRouter.delete("/snooze", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const alertId = typeof req.query.alertId === "string" ? req.query.alertId : "";
|
||||
if (!alertId) return res.status(400).json({ error: "invalid_id" });
|
||||
const removed = await clearSnooze(alertId);
|
||||
if (!removed) return res.status(404).json({ error: "not_found" });
|
||||
const alert = (await getAlerts(false)).alerts.find((a) => a.id === alertId);
|
||||
await recordAudit({ actor: req.currentUser!, category: "alert", action: "unsnooze", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: removed.message } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -0,0 +1,27 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { storageHistory, storageOverview } from "../services/storageHistory.js";
|
||||
import { bucketPoints } from "../services/serverMetrics.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
/** The pools Proxmox and Synology report, with how fast each is filling: for everyone signed in, like the pages that show the pools. */
|
||||
export const storageRouter = Router();
|
||||
storageRouter.use(requireAuth);
|
||||
|
||||
storageRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const { healthChecks } = await getSettings();
|
||||
res.json({ pools: await storageOverview(), forecastDays: healthChecks.diskForecastDays });
|
||||
}));
|
||||
|
||||
// One pool's readings, for the chart: up to the 400 days that are kept, thinned to a drawable number of points.
|
||||
storageRouter.get("/:id/history", asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const asked = Number(req.query.days);
|
||||
const days = Number.isFinite(asked) ? Math.min(400, Math.max(1, Math.round(asked))) : 30;
|
||||
const to = Date.now();
|
||||
const from = to - days * 86_400_000;
|
||||
const rows = await storageHistory(id, days, new Date(to));
|
||||
res.json({ days, points: bucketPoints(rows.map((r) => ({ t: Date.parse(r.t), v: r.v })), from, to, 120) });
|
||||
}));
|
||||
@@ -74,6 +74,7 @@ export function mostWithin(times: number[], windowMs: number): number {
|
||||
/** What to try, from the kind of alert and the key it has. Plain advice; nothing it says is done for anyone. */
|
||||
export function hintFor(category: string, alertId: string): string {
|
||||
const key = keyOfAlertId(alertId, category);
|
||||
if (key.startsWith("rule:")) return "A limit of your own (Monitoring → Alert rules): if it keeps coming back, raise the limit or make it hold for longer before it raises.";
|
||||
if (key.startsWith("new-port:")) return "If this port should be listening, mark it as expected on the server's Ports card.";
|
||||
if (key.startsWith("heartbeat")) return "If the job is just sometimes late, give its heartbeat more grace; if it fails, look at what it printed.";
|
||||
if (key.startsWith("endpoint:")) return "Raise “Alert after … failures in a row” on the check, or its timeout or interval — or fix what is making it fail.";
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
/**
|
||||
* Alert rules of your own: "CPU above 90% for 15 minutes", "memory above 85%", "a disk is warmer than 55 °C". Each is a limit on one figure of the servers it covers (all of
|
||||
* them, one, or those with a tag), with the severity it should have. They are judged on the same pass as the built-in checks — what they find becomes ordinary health
|
||||
* conditions (`rule:<id>:server:<server>[:<disk>]`) — so they are announced, recovered, held back while flapping, silenced by maintenance and routed like the rest.
|
||||
*
|
||||
* Load, memory and disk use are judged from the history the agents' reports build (a sample every ten to fifteen minutes): a rule with a time says the figure must have been
|
||||
* past its limit on every sample for that long. A disk's temperature is only known as of its last report, so it is judged right now. A server that is not reporting is not
|
||||
* judged — its own offline alert says that.
|
||||
*/
|
||||
import { eq, gte } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { alertRules, serverMetrics, servers, type AlertRuleMetric } from "../db/schema.js";
|
||||
import { parseHostHealth } from "./hostHealth.js";
|
||||
import { InvalidTagError, normalizeTag, parseTags } from "./serverTags.js";
|
||||
|
||||
export type RuleRow = typeof alertRules.$inferSelect;
|
||||
|
||||
export const MAX_FOR_MINUTES = 24 * 60;
|
||||
export const MAX_RULES = 100;
|
||||
/** A server whose newest sample is older than this is not judged: the figure is no longer "now". */
|
||||
export const STALE_AFTER_MS = 45 * 60_000;
|
||||
|
||||
export const METRIC_WORDS: Record<AlertRuleMetric, { label: string; unit: string }> = {
|
||||
cpu: { label: "CPU load", unit: "%" },
|
||||
mem: { label: "Memory in use", unit: "%" },
|
||||
disk: { label: "Disk use", unit: "%" },
|
||||
disk_temp: { label: "Disk temperature", unit: " °C" },
|
||||
};
|
||||
|
||||
export interface RuleFinding {
|
||||
ruleId: number;
|
||||
ruleName: string;
|
||||
serverId: number;
|
||||
serverName: string;
|
||||
/** The disk (mount point, or device for a temperature) it is about, if any. */
|
||||
detail: string | null;
|
||||
value: number;
|
||||
severity: "warning" | "critical";
|
||||
/** When this run of samples past the limit began (epoch ms); null when it is judged right now. */
|
||||
since: number | null;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface RuleServer {
|
||||
id: number;
|
||||
name: string;
|
||||
tags: string[];
|
||||
/** What the last report said about each disk's temperature. */
|
||||
smart: { device: string; temperatureC: number | null }[];
|
||||
}
|
||||
|
||||
export type RuleSamples = Map<string, { t: number; v: number }[]>;
|
||||
export const sampleKey = (serverId: number, metric: string) => `${serverId}|${metric}`;
|
||||
|
||||
const durationWords = (minutes: number) => (minutes < 60 ? `${minutes} min` : minutes % 60 === 0 ? `${minutes / 60} h` : `${Math.floor(minutes / 60)} h ${minutes % 60} min`);
|
||||
|
||||
const breaches = (rule: Pick<RuleRow, "operator" | "threshold">, v: number) => (rule.operator === "above" ? v > rule.threshold : v < rule.threshold);
|
||||
|
||||
export function ruleCovers(rule: Pick<RuleRow, "serverId" | "tag">, server: Pick<RuleServer, "id" | "tags">): boolean {
|
||||
if (rule.serverId !== null) return rule.serverId === server.id;
|
||||
if (rule.tag) return server.tags.some((t) => t.toLowerCase() === rule.tag!.toLowerCase());
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: how long the newest samples have been past the limit without a break. Returns the figure now and when the unbroken run began, or null when the newest sample is
|
||||
* not past the limit (or is too old to say).
|
||||
*/
|
||||
export function currentRun(rule: Pick<RuleRow, "operator" | "threshold">, samples: { t: number; v: number }[], now: number): { value: number; since: number; latest: number } | null {
|
||||
if (samples.length === 0) return null;
|
||||
const sorted = [...samples].sort((a, b) => a.t - b.t);
|
||||
const newest = sorted[sorted.length - 1];
|
||||
if (now - newest.t > STALE_AFTER_MS || !breaches(rule, newest.v)) return null;
|
||||
let since = newest.t;
|
||||
for (let i = sorted.length - 2; i >= 0; i--) {
|
||||
if (!breaches(rule, sorted[i].v)) break;
|
||||
since = sorted[i].t;
|
||||
}
|
||||
return { value: newest.v, since, latest: newest.t };
|
||||
}
|
||||
|
||||
/** Pure: what the rules find on these servers, from their samples (and, for temperatures, the last report). */
|
||||
export function evaluateRules(rules: RuleRow[], serversIn: RuleServer[], samples: RuleSamples, now: number): RuleFinding[] {
|
||||
const out: RuleFinding[] = [];
|
||||
for (const rule of rules) {
|
||||
if (!rule.enabled) continue;
|
||||
const word = METRIC_WORDS[rule.metric];
|
||||
const limit = `${rule.operator} ${rule.threshold}${word.unit}`;
|
||||
for (const s of serversIn) {
|
||||
if (!ruleCovers(rule, s)) continue;
|
||||
if (rule.metric === "disk_temp") {
|
||||
for (const d of s.smart) {
|
||||
if (d.temperatureC === null || !breaches(rule, d.temperatureC)) continue;
|
||||
out.push({ ruleId: rule.id, ruleName: rule.name, serverId: s.id, serverName: s.name, detail: d.device, value: d.temperatureC, severity: rule.severity, since: null, message: `${s.name}: ${d.device} is ${Math.round(d.temperatureC)}${word.unit}, ${limit} [${rule.name}]` });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const metrics = rule.metric === "disk" ? [...samples.keys()].filter((k) => k.startsWith(`${s.id}|disk:`) && (rule.mount === null || k === sampleKey(s.id, `disk:${rule.mount}`))) : [sampleKey(s.id, rule.metric)];
|
||||
for (const k of metrics) {
|
||||
const run = currentRun(rule, samples.get(k) ?? [], now);
|
||||
if (!run) continue;
|
||||
// The figure must have been past the limit for the whole time: the run began that long before the newest sample.
|
||||
const heldFor = Math.floor((run.latest - run.since) / 60_000);
|
||||
if (rule.forMinutes > 0 && heldFor < rule.forMinutes) continue;
|
||||
const detail = rule.metric === "disk" ? k.slice(`${s.id}|disk:`.length) : null;
|
||||
const what = rule.metric === "disk" ? `${s.name}: ${detail} use` : `${s.name}: ${word.label.toLowerCase()}`;
|
||||
out.push({
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
serverId: s.id,
|
||||
serverName: s.name,
|
||||
detail,
|
||||
value: run.value,
|
||||
severity: rule.severity,
|
||||
since: run.since,
|
||||
message: `${what} is ${Math.round(run.value)}${word.unit}, ${limit}${rule.forMinutes > 0 ? ` for ${durationWords(rule.forMinutes)}` : ""} [${rule.name}]`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The condition key a finding has: the same finding always has the same key, so it is announced once. */
|
||||
export const findingKey = (f: Pick<RuleFinding, "ruleId" | "serverId" | "detail">) => `rule:${f.ruleId}:server:${f.serverId}${f.detail ? `:${f.detail}` : ""}`;
|
||||
|
||||
export async function listRules(): Promise<RuleRow[]> {
|
||||
return (await db.select().from(alertRules)).sort((a, b) => a.name.localeCompare(b.name) || a.id - b.id);
|
||||
}
|
||||
|
||||
/** What the given rules find right now, from the stored history. Reads nothing when no rule is on. */
|
||||
export async function computeRuleFindings(rules: RuleRow[], now: number = Date.now()): Promise<RuleFinding[]> {
|
||||
const on = rules.filter((r) => r.enabled);
|
||||
if (on.length === 0) return [];
|
||||
const rows = await db.select({ id: servers.id, name: servers.name, tags: servers.tags, hostHealth: servers.hostHealth }).from(servers);
|
||||
const serverList: RuleServer[] = rows.map((r) => ({
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
tags: parseTags(r.tags),
|
||||
smart: (parseHostHealth(r.hostHealth)?.smart ?? []).map((d) => ({ device: d.device, temperatureC: d.temperatureC })),
|
||||
}));
|
||||
const samples: RuleSamples = new Map();
|
||||
if (on.some((r) => r.metric !== "disk_temp")) {
|
||||
const longest = Math.max(...on.filter((r) => r.metric !== "disk_temp").map((r) => r.forMinutes));
|
||||
const since = new Date(now - (longest * 60_000 + 3 * 3600_000)).toISOString();
|
||||
for (const r of await db.select().from(serverMetrics).where(gte(serverMetrics.recordedAt, since))) {
|
||||
if (r.metric !== "cpu" && r.metric !== "mem" && !r.metric.startsWith("disk:")) continue;
|
||||
const k = sampleKey(r.serverId, r.metric);
|
||||
const list = samples.get(k) ?? [];
|
||||
list.push({ t: Date.parse(r.recordedAt), v: r.value });
|
||||
samples.set(k, list);
|
||||
}
|
||||
}
|
||||
return evaluateRules(on, serverList, samples, now);
|
||||
}
|
||||
|
||||
// ─── Making and changing rules ─────────────────────────────────────────────
|
||||
|
||||
export class RuleError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly code: string,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export interface RuleInput {
|
||||
name: string;
|
||||
enabled?: boolean;
|
||||
metric: AlertRuleMetric;
|
||||
mount?: string | null;
|
||||
operator?: "above" | "below";
|
||||
threshold: number;
|
||||
forMinutes?: number;
|
||||
severity?: "warning" | "critical";
|
||||
serverId?: number | null;
|
||||
tag?: string | null;
|
||||
}
|
||||
|
||||
/** Checks and tidies what was asked for; says plainly what is wrong with it. */
|
||||
export async function checkedRule(input: RuleInput): Promise<Omit<RuleRow, "id" | "createdAt" | "createdBy">> {
|
||||
const name = input.name.trim();
|
||||
if (name === "" || name.length > 80) throw new RuleError(400, "invalid_name", "Give the rule a name, up to 80 characters.");
|
||||
const operator = input.operator ?? "above";
|
||||
if (!Number.isFinite(input.threshold)) throw new RuleError(400, "invalid_threshold", "The limit has to be a number.");
|
||||
const percent = input.metric !== "disk_temp";
|
||||
if (percent && (input.threshold < 0 || input.threshold > 100)) throw new RuleError(400, "invalid_threshold", "A percentage limit is between 0 and 100.");
|
||||
if (!percent && (input.threshold < 0 || input.threshold > 150)) throw new RuleError(400, "invalid_threshold", "A temperature limit is between 0 and 150 °C.");
|
||||
const forMinutes = percent ? Math.round(input.forMinutes ?? 0) : 0;
|
||||
if (!Number.isInteger(forMinutes) || forMinutes < 0 || forMinutes > MAX_FOR_MINUTES) throw new RuleError(400, "invalid_for", "How long it has to hold is from 0 to 1440 minutes (a day).");
|
||||
const mount = input.metric === "disk" ? input.mount?.trim() || null : null;
|
||||
if (mount && mount.length > 200) throw new RuleError(400, "invalid_mount", "That disk name is too long.");
|
||||
let tag: string | null = null;
|
||||
if (input.tag?.trim()) {
|
||||
try {
|
||||
tag = normalizeTag(input.tag);
|
||||
} catch (err) {
|
||||
if (err instanceof InvalidTagError) throw new RuleError(400, "invalid_tag", err.message);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
const serverId = input.serverId ?? null;
|
||||
if (serverId !== null && tag) throw new RuleError(400, "invalid_scope", "A rule is for one server, or for servers with a tag, not both.");
|
||||
if (serverId !== null) {
|
||||
const [row] = await db.select({ id: servers.id }).from(servers).where(eq(servers.id, serverId)).limit(1);
|
||||
if (!row) throw new RuleError(400, "unknown_server", "That server is not there.");
|
||||
}
|
||||
return { name, enabled: input.enabled ?? true, metric: input.metric, mount, operator, threshold: input.threshold, forMinutes, severity: input.severity ?? "warning", serverId, tag };
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/**
|
||||
* Snooze: "remind me tomorrow at 09:00". An alert that is snoozed is hidden — on the Alerts page it is held back like one under maintenance, so it is not counted in the
|
||||
* badges, the dashboard or the digest — and the messages about it from the health and automation checks are not sent. It is not maintenance: nothing is promised about
|
||||
* the thing, only that nobody wants to hear about it until then. When the time comes, an alert that is still there is announced again, once, as a reminder; one that
|
||||
* went away in the meantime is simply forgotten.
|
||||
*
|
||||
* A snooze belongs to the alert's id and goes away, like an acknowledgement, once the alert has been gone for half an hour — the same problem coming back later is new.
|
||||
*/
|
||||
import { eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { alertSnoozes } from "../db/schema.js";
|
||||
|
||||
export const MAX_DAYS = 90;
|
||||
/** At least this far ahead, so a snooze cannot end before it has begun. */
|
||||
export const MIN_AHEAD_MS = 60_000;
|
||||
/** A snoozed alert that has been absent this long loses its snooze. */
|
||||
export const GONE_AFTER_MS = 30 * 60_000;
|
||||
|
||||
export class SnoozeError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly code: string,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export interface SnoozeInfo {
|
||||
until: string;
|
||||
snoozedBy: string;
|
||||
snoozedAt: string;
|
||||
}
|
||||
|
||||
const shape = (r: typeof alertSnoozes.$inferSelect): SnoozeInfo => ({ until: r.until, snoozedBy: r.snoozedBy, snoozedAt: r.snoozedAt });
|
||||
|
||||
/** The snoozes still running, by alert id. */
|
||||
export async function loadSnoozes(now: Date = new Date()): Promise<Map<string, SnoozeInfo>> {
|
||||
return new Map((await db.select().from(alertSnoozes)).filter((r) => Date.parse(r.until) > now.getTime()).map((r) => [r.alertId, shape(r)]));
|
||||
}
|
||||
|
||||
/** Pure: whether a notification about this condition key is held because it is snoozed. */
|
||||
export function isSnoozedKey(rows: { conditionKey: string; until: string }[], key: string | undefined, now: Date): boolean {
|
||||
return key !== undefined && rows.some((r) => r.conditionKey === key && Date.parse(r.until) > now.getTime());
|
||||
}
|
||||
|
||||
/** Drops from a list of messages the ones about alerts that are snoozed. */
|
||||
export async function withoutSnoozed<T extends { key?: string }>(items: T[], now: Date = new Date()): Promise<T[]> {
|
||||
if (!items.some((i) => i.key)) return items;
|
||||
const rows = await db.select({ conditionKey: alertSnoozes.conditionKey, until: alertSnoozes.until }).from(alertSnoozes);
|
||||
if (rows.length === 0) return items;
|
||||
return items.filter((i) => !isSnoozedKey(rows, i.key, now));
|
||||
}
|
||||
|
||||
export async function setSnooze(input: { alertId: string; conditionKey: string; message: string; until: Date; actor: string }, now: Date = new Date()): Promise<SnoozeInfo> {
|
||||
if (!(input.until.getTime() >= now.getTime() + MIN_AHEAD_MS)) throw new SnoozeError(400, "invalid_until", "Choose a time that has not come yet.");
|
||||
if (input.until.getTime() > now.getTime() + MAX_DAYS * 86_400_000) throw new SnoozeError(400, "invalid_until", `A snooze can be at most ${MAX_DAYS} days.`);
|
||||
const values = { conditionKey: input.conditionKey, message: input.message.slice(0, 500), until: input.until.toISOString(), snoozedBy: input.actor, snoozedAt: now.toISOString(), lastSeenAt: now.toISOString() };
|
||||
await db.insert(alertSnoozes).values({ alertId: input.alertId, ...values }).onConflictDoUpdate({ target: alertSnoozes.alertId, set: values });
|
||||
return { until: values.until, snoozedBy: values.snoozedBy, snoozedAt: values.snoozedAt };
|
||||
}
|
||||
|
||||
export async function clearSnooze(alertId: string): Promise<{ message: string } | null> {
|
||||
const [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, alertId)).limit(1);
|
||||
if (!row) return null;
|
||||
await db.delete(alertSnoozes).where(eq(alertSnoozes.id, row.id));
|
||||
return { message: row.message };
|
||||
}
|
||||
|
||||
export interface Reminder {
|
||||
key: string;
|
||||
message: string;
|
||||
severity?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* One look, with the alerts that are there now: snoozes that have run out are removed, and each whose alert is still there becomes a reminder; the others' alerts are
|
||||
* marked as seen, and those that have been gone for half an hour are dropped. Returns the reminders to send.
|
||||
*/
|
||||
export async function settleSnoozes(present: { id: string; message: string; severity: string; key: string }[], now: Date = new Date()): Promise<Reminder[]> {
|
||||
const rows = await db.select().from(alertSnoozes);
|
||||
if (rows.length === 0) return [];
|
||||
const byId = new Map(present.map((a) => [a.id, a]));
|
||||
const reminders: Reminder[] = [];
|
||||
const done: number[] = [];
|
||||
const seen: number[] = [];
|
||||
const cutoff = new Date(now.getTime() - GONE_AFTER_MS).toISOString();
|
||||
for (const r of rows) {
|
||||
const there = byId.get(r.alertId);
|
||||
if (Date.parse(r.until) <= now.getTime()) {
|
||||
if (there) reminders.push({ key: r.conditionKey, message: there.message, severity: there.severity });
|
||||
done.push(r.id);
|
||||
} else if (there) seen.push(r.id);
|
||||
else if (r.lastSeenAt < cutoff) done.push(r.id);
|
||||
}
|
||||
if (seen.length > 0) await db.update(alertSnoozes).set({ lastSeenAt: now.toISOString() }).where(inArray(alertSnoozes.id, seen));
|
||||
if (done.length > 0) await db.delete(alertSnoozes).where(inArray(alertSnoozes.id, done));
|
||||
return reminders;
|
||||
}
|
||||
@@ -23,6 +23,8 @@ export interface Alert {
|
||||
causedBy?: { name: string; alertId?: string };
|
||||
/** On a host's alert: the servers on it whose own alerts are folded under this one. */
|
||||
affects?: string[];
|
||||
/** Snoozed until then (services/alertSnoozes.ts): held back like one under maintenance, and announced again if it is still there. */
|
||||
snoozed?: { until: string; by: string };
|
||||
/** Raised and cleared over and over: its raised and cleared messages are being held back (services/flapGuard.ts). */
|
||||
flapping?: boolean;
|
||||
/** What a maintenance window would have to be on to silence it, when there is such a thing — which is what "Silence" acts on. */
|
||||
|
||||
@@ -13,6 +13,7 @@ import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, haEntities, haWatched, heartbeats, integrations, networkDevices, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
||||
import { storageFillingSoon, storageOverview } from "./storageHistory.js";
|
||||
import { daysOfSupportLeft, lifecycleOf } from "./osLifecycle.js";
|
||||
import { npmIntegrationIds, npmProblems, readNpm } from "./npmReader.js";
|
||||
import { haProblems, haSyncStatuses, isHaFresh } from "./homeAssistantSync.js";
|
||||
@@ -21,6 +22,7 @@ import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
import { recordFromReport } from "./incidents.js";
|
||||
import { reconcileAcks } from "./alertAcks.js";
|
||||
import { flappingKeys, keyOfAlertId } from "./flapGuard.js";
|
||||
import { loadSnoozes } from "./alertSnoozes.js";
|
||||
import { lastAttempt as lastSelfBackup, newestSnapshot, STALE_AFTER_HOURS as SELF_BACKUP_STALE_HOURS } from "./selfBackup.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
|
||||
@@ -88,7 +90,9 @@ const SEVERITY_ORDER: Record<AlertSeverity, number> = { critical: 0, warning: 1,
|
||||
function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } {
|
||||
const device = /^offline:device:(\d+)$/.exec(key);
|
||||
if (device) return { type: "unifi", id: Number(device[1]), category: "offline" };
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
|
||||
let m = /^rule:\d+:server:(\d+)/.exec(key);
|
||||
if (m) return { type: "server", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
|
||||
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : key.startsWith("new-port") ? "monitoring" : "disk" };
|
||||
m = /^disk:proxmox:(\d+):/.exec(key);
|
||||
if (m) return { type: "proxmox", id: Number(m[1]), category: "disk" };
|
||||
@@ -217,11 +221,12 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : ""}` : origin.type === "unifi" ? "/network/devices" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
|
||||
if (held.has("server:rules")) cannotCheck("Alert rules", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
@@ -520,6 +525,26 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
}
|
||||
}),
|
||||
|
||||
// The same for the pools the integrations report (a Proxmox storage, a Synology volume), from the history kept on each health pass.
|
||||
check("Storage forecasts", async () => {
|
||||
const { healthChecks } = await getSettings();
|
||||
if (!(healthChecks.diskForecastDays > 0)) return;
|
||||
const nowMs = Date.now();
|
||||
const rows = (await storageOverview()).filter((r) => nowMs - Date.parse(r.updatedAt) < 2 * 86_400_000 && intById.get(r.integrationId)?.enabled);
|
||||
for (const f of storageFillingSoon(rows, healthChecks.diskForecastDays, healthChecks.diskUsagePercent)) {
|
||||
const row = intById.get(f.integrationId)!;
|
||||
push({
|
||||
category: "disk",
|
||||
severity: f.daysToFull <= 3 ? "critical" : "warning",
|
||||
type: row.type,
|
||||
message: `${f.label} is filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")} (${Math.round(f.latest)}% used now)`,
|
||||
key: `storage-forecast:${f.integrationId}:${f.key}`,
|
||||
silenced: silencedIntegration(f.integrationId),
|
||||
target: integrationTarget(f.integrationId),
|
||||
});
|
||||
}
|
||||
}),
|
||||
|
||||
// Certificates in use by Nginx Proxy Manager that are running out, and hosts it could not set up.
|
||||
check("Nginx Proxy Manager", async () => {
|
||||
for (const i of await npmIntegrationIds()) {
|
||||
@@ -617,6 +642,18 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const flapping = await flappingKeys().catch(() => new Set<string>());
|
||||
if (flapping.size > 0) for (const a of alerts) if (flapping.has(keyOfAlertId(a.id, a.category))) a.flapping = true;
|
||||
|
||||
// Those somebody has snoozed are held back like those under maintenance, with when they come back.
|
||||
const snoozes = await loadSnoozes().catch(() => new Map<string, { until: string; snoozedBy: string }>());
|
||||
if (snoozes.size > 0) {
|
||||
for (const a of alerts) {
|
||||
const sn = snoozes.get(a.id);
|
||||
if (sn) {
|
||||
a.snoozed = { until: sn.until, by: sn.snoozedBy };
|
||||
a.silenced = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const active = alerts.filter((a) => !a.silenced);
|
||||
const grouped = alerts.filter((a) => a.causedBy).length;
|
||||
return {
|
||||
|
||||
@@ -40,6 +40,7 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
|
||||
share_link: "settings",
|
||||
metrics_token: "settings",
|
||||
api_token: "settings",
|
||||
alert_rule: "alerts",
|
||||
todo: "other",
|
||||
note: "servers",
|
||||
webhook: "settings",
|
||||
@@ -178,6 +179,19 @@ export function describeChange(category: string, action: string, detail: Record<
|
||||
return { title: "Acknowledging several alerts at once was undone", tone: "neutral", detail: null };
|
||||
case "inventory/bulk_change":
|
||||
return { title: detail.change === "undo" ? "A change to several pieces of hardware was undone" : `${typeof detail.changed === "number" ? detail.changed : "Several"} piece${detail.changed === 1 ? "" : "s"} of hardware changed at once`, tone: "neutral", detail: genericSummary(detail, ["ids", "changed", "change"]) };
|
||||
case "alert_rule/create":
|
||||
return { title: `Alert rule added${name ? `: ${name}` : ""}`, tone: "neutral", detail: genericSummary(detail) };
|
||||
case "alert_rule/update":
|
||||
return { title: `Alert rule changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: genericSummary(detail) };
|
||||
case "alert_rule/enable":
|
||||
case "alert_rule/disable":
|
||||
return { title: `Alert rule switched ${action === "enable" ? "on" : "off"}${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "alert_rule/delete":
|
||||
return { title: `Alert rule removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "alert/snooze":
|
||||
return { title: `Alert snoozed${typeof detail.message === "string" ? `: ${detail.message}` : ""}`, tone: "neutral", detail: str(detail.until) ? `until ${str(detail.until)}` : null };
|
||||
case "alert/unsnooze":
|
||||
return { title: `Snooze ended early${typeof detail.message === "string" ? `: ${detail.message}` : ""}`, tone: "neutral", detail: null };
|
||||
case "alert/flapping_started":
|
||||
return { title: `Alert began flapping${str(detail.message) ? `: ${str(detail.message)}` : ""}`, tone: "bad", detail: typeof detail.raises === "number" ? `raised ${detail.raises} times within an hour; its messages are held back` : null };
|
||||
case "alert/flapping_ended":
|
||||
|
||||
@@ -15,6 +15,8 @@ import { hostHealthConditions, parseHostHealth, withoutIgnoredServices, type Hos
|
||||
import { newListeningPorts, newPortMessage, parseBaseline, portKey, type NewPort } from "./newPorts.js";
|
||||
import { parseJson, type AgentPort } from "./agentPorts.js";
|
||||
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
|
||||
import { computeRuleFindings, findingKey, listRules, type RuleFinding } from "./alertRules.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
|
||||
const STATE_FLAG = HEALTH_STATE_FLAG;
|
||||
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
|
||||
@@ -66,6 +68,8 @@ export interface HealthSnapshot {
|
||||
proxmox: ProxmoxSnapshot[];
|
||||
synology: SynologySnapshot[];
|
||||
unifi?: UnifiSnapshot[];
|
||||
/** What the administrator's own alert rules find on the servers (services/alertRules.ts). */
|
||||
rules?: RuleFinding[];
|
||||
}
|
||||
export interface Thresholds {
|
||||
serverOfflineMinutes: number;
|
||||
@@ -155,6 +159,15 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
}
|
||||
}
|
||||
|
||||
// The administrator's own rules: judged only for servers that are reporting (one that is not has its offline alert).
|
||||
if (!opts.skipServers) {
|
||||
const offlineIds = new Set(out.map((c) => /^offline:server:(\d+)$/.exec(c.key)?.[1]).filter((id): id is string => id !== undefined).map(Number));
|
||||
for (const f of snapshot.rules ?? []) {
|
||||
if (offlineIds.has(f.serverId)) continue;
|
||||
out.push({ key: findingKey(f), source: "server:rules", message: f.message, severity: f.severity });
|
||||
}
|
||||
}
|
||||
|
||||
for (const u of snapshot.unifi ?? []) {
|
||||
const limitMs = (thresholds.deviceOfflineMinutes ?? 10) * 60_000;
|
||||
for (const d of u.devices) {
|
||||
@@ -351,6 +364,14 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: withoutIgnoredServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.ignoreServices), thresholds: parseThresholds(s.thresholds), newPorts });
|
||||
}
|
||||
|
||||
// Their own rules: what the stored history says. A failure here costs only the rules.
|
||||
try {
|
||||
snapshot.rules = await computeRuleFindings(await listRules());
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge the alert rules:", err instanceof Error ? err.message : err);
|
||||
held.add("server:rules");
|
||||
}
|
||||
|
||||
const rows = await db
|
||||
.select({ id: integrations.id, name: integrations.name, type: integrations.type })
|
||||
.from(integrations)
|
||||
@@ -446,6 +467,8 @@ export function alsoList(names: string[]): string {
|
||||
export async function runHealthCheck(now: number = Date.now()): Promise<{ added: number; resolved: number }> {
|
||||
const { healthChecks } = await getSettings();
|
||||
const { snapshot, held } = await collectSnapshot();
|
||||
// What the pools read this time go into their history (one reading an hour at most), so they can be forecast.
|
||||
await recordStorageSamples(snapshot, new Date(now));
|
||||
|
||||
const inGrace = now - PROCESS_START < STARTUP_GRACE_MS;
|
||||
if (inGrace) held.add("server");
|
||||
|
||||
@@ -25,6 +25,8 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
|
||||
export function subjectOfConditionKey(key: string): string | null {
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)(?::|$)/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^rule:\d+:server:(\d+)(?::|$)/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^offline:device:(\d+)$/.exec(key);
|
||||
|
||||
@@ -20,6 +20,7 @@ import { lastAttempt as lastSelfBackup, listSnapshots } from "./selfBackup.js";
|
||||
import { loadAcks, attentionCounts } from "./alertAcks.js";
|
||||
import { listIncidents } from "./incidents.js";
|
||||
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
||||
import { storageFillingSoon, storageOverview } from "./storageHistory.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
@@ -139,6 +140,13 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
} catch (err) {
|
||||
console.error("[metrics] couldn't work out the disk forecasts:", err);
|
||||
}
|
||||
try {
|
||||
for (const f of storageFillingSoon(await storageOverview(now), 3650, 101)) {
|
||||
out.add("homelab_storage_days_to_full", "gauge", "Days until a Proxmox storage or Synology volume is full at its present rate of filling; only for those with a steady climb.", f.daysToFull, { pool: f.label });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[metrics] couldn't work out the storage forecasts:", err);
|
||||
}
|
||||
try {
|
||||
const uptime = await getUptime(now, 30);
|
||||
for (const s of serverRows) {
|
||||
|
||||
@@ -8,6 +8,7 @@ import { inArray } from "drizzle-orm";
|
||||
import { pushSubscriptions, servers } from "../db/schema.js";
|
||||
import { parseTags } from "./serverTags.js";
|
||||
import { dampRaised, dampRecovered } from "./flapGuard.js";
|
||||
import { withoutSnoozed } from "./alertSnoozes.js";
|
||||
import { firstMatchingRule, metaOfConditionKey, pushAccepts, type ChannelId, type NotifyMeta } from "./notifyRouting.js";
|
||||
|
||||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||||
@@ -484,6 +485,9 @@ export async function notifyHealthIssues(issues: { message: string; key?: string
|
||||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||||
if (issues.length === 0) return;
|
||||
// One somebody has snoozed is not announced now; if it is still there when the snooze ends, it is announced then (services/alertSnoozes.ts).
|
||||
issues = await withoutSnoozed(issues);
|
||||
if (issues.length === 0) return;
|
||||
// An alert that keeps coming and going is announced once as "flapping" and then left quiet (services/flapGuard.ts); that is tracked whether or not this is switched on.
|
||||
const { pass, started } = await dampRaised(issues);
|
||||
if (!(await eventEnabled("healthAlerts"))) return;
|
||||
@@ -499,6 +503,8 @@ export async function notifyHealthIssues(issues: { message: string; key?: string
|
||||
export async function notifyHealthRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||||
if (resolved.length === 0) return;
|
||||
resolved = await withoutSnoozed(resolved);
|
||||
if (resolved.length === 0) return;
|
||||
const passed = await dampRecovered(resolved);
|
||||
if (!(await eventEnabled("healthAlerts"))) return;
|
||||
if (passed.length === 0) return;
|
||||
@@ -509,6 +515,15 @@ export async function notifyHealthRecovered(resolved: { message: string; key?: s
|
||||
}));
|
||||
}
|
||||
|
||||
/** Alerts that were snoozed, whose time is up and that are still there: said once, as a reminder. */
|
||||
export async function notifySnoozeEnded(items: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||||
if (items.length === 0) return;
|
||||
await notifyGrouped(items, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Snooze ended",
|
||||
message: `${group.length === 1 ? "An alert you snoozed is" : `${group.length} alerts you snoozed are`} still there:\n\n${group.map((i) => `⏰ ${i.message}`).join("\n")}`,
|
||||
}));
|
||||
}
|
||||
|
||||
/** An alert that has just been called flapping: said once, with how often it has come, instead of each time. */
|
||||
export async function notifyFlapping(items: { message: string; key?: string; severity?: string; count: number }[]): Promise<void> {
|
||||
if (items.length === 0) return;
|
||||
@@ -531,7 +546,8 @@ export async function notifyAutomationFailed(issues: { message: string; key?: st
|
||||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||||
if (issues.length === 0) return;
|
||||
if (!(await eventEnabled("automationAlerts"))) return;
|
||||
issues = await withoutSnoozed(issues);
|
||||
if (issues.length === 0 || !(await eventEnabled("automationAlerts"))) return;
|
||||
await notifyGrouped(issues, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Automation Failed",
|
||||
message: `${group.length} run${group.length !== 1 ? "s" : ""} failed:\n\n${group.map((i) => `✕ ${i.message}`).join("\n")}`,
|
||||
@@ -541,7 +557,8 @@ export async function notifyAutomationFailed(issues: { message: string; key?: st
|
||||
export async function notifyAutomationRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||||
if (resolved.length === 0) return;
|
||||
if (!(await eventEnabled("automationAlerts"))) return;
|
||||
resolved = await withoutSnoozed(resolved);
|
||||
if (resolved.length === 0 || !(await eventEnabled("automationAlerts"))) return;
|
||||
await notifyGrouped(resolved, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Automation Recovered",
|
||||
message: `${group.length} no longer failing:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||||
|
||||
@@ -108,11 +108,12 @@ export function metaOfConditionKey(key: string, severity?: string): NotifyMeta {
|
||||
if (key.startsWith("offline:")) category = "offline";
|
||||
else if (key.startsWith("failed-units:")) category = "services";
|
||||
else if (key.startsWith("new-port:")) category = "monitoring";
|
||||
else if (key.startsWith("rule:")) category = "monitoring";
|
||||
else if (/^(disk|smart|zfs|synology-volume|synology-disk):/.test(key)) category = "disk";
|
||||
else if (key.startsWith("automation:")) category = "automation";
|
||||
else if (key.startsWith("heartbeat")) category = "automation";
|
||||
else if (key.startsWith("endpoint-cert:")) category = "expiry";
|
||||
else if (key.startsWith("endpoint:")) category = "monitoring";
|
||||
const server = /^(?:(?:offline|disk|failed-units|smart|zfs|new-port):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
|
||||
const server = /^(?:(?:offline|disk|failed-units|smart|zfs|new-port|rule:\d+):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
|
||||
return { category, severity: severity === "critical" ? "critical" : "warning", serverId: server ? Number(server[1]) : null };
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Every minute: snoozes that have run out are looked at (see services/alertSnoozes.ts) — an alert that is still there is announced again, once; one that has gone is forgotten.
|
||||
*/
|
||||
import { getAlerts, invalidateAlertsCache } from "./alerts.js";
|
||||
import { keyOfAlertId } from "./flapGuard.js";
|
||||
import { settleSnoozes } from "./alertSnoozes.js";
|
||||
import { notifySnoozeEnded } from "./notify.js";
|
||||
|
||||
/** One look: returns how many reminders were sent. */
|
||||
export async function remindAboutSnoozed(now: Date = new Date()): Promise<number> {
|
||||
const report = await getAlerts(false);
|
||||
const reminders = await settleSnoozes(report.alerts.map((a) => ({ id: a.id, message: a.message, severity: a.severity, key: keyOfAlertId(a.id, a.category) })), now);
|
||||
if (reminders.length > 0) {
|
||||
// The page should show them again at once, not when its stored result expires.
|
||||
invalidateAlertsCache();
|
||||
await notifySnoozeEnded(reminders);
|
||||
}
|
||||
return reminders.length;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initSnoozeMonitor(): void {
|
||||
if (timer) return;
|
||||
timer = setInterval(() => void remindAboutSnoozed().catch((err) => console.error("[snooze] couldn't look at the snoozed alerts:", err)), 60_000);
|
||||
timer.unref();
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* A history for the pools the integrations report — a Proxmox storage or root filesystem, a Synology volume — so they can be forecast the way an agent server's disk is
|
||||
* ("this storage is full in 40 days"). The big disks of a homelab are usually these, not the machines the agents run on.
|
||||
*
|
||||
* Nothing new is read: on each scheduled health pass the figures it has just read are written down, one reading an hour at most, and kept for 400 days. A pool that stops
|
||||
* being reported for a month is forgotten. The forecast itself is the same straight-line fit as for a server's disks (services/serverMetrics.ts): enough history, steady
|
||||
* growth, and not already past the limit the ordinary "disk nearly full" alert uses.
|
||||
*/
|
||||
import { and, desc, eq, gte, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { storageSamples, storageSeries } from "../db/schema.js";
|
||||
import type { HealthSnapshot } from "./healthMonitor.js";
|
||||
import { disksFillingSoon, trendOf, type MetricSeries, type Trend } from "./serverMetrics.js";
|
||||
|
||||
/** Readings closer together than this add no sample. */
|
||||
export const MIN_GAP_MS = 55 * 60_000;
|
||||
export const KEEP_DAYS = 400;
|
||||
/** A pool not reported for this long is dropped, with its history. */
|
||||
export const FORGET_AFTER_DAYS = 30;
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
export interface PoolReading {
|
||||
integrationId: number;
|
||||
key: string;
|
||||
label: string;
|
||||
percent: number;
|
||||
usedBytes: number | null;
|
||||
totalBytes: number | null;
|
||||
}
|
||||
|
||||
const pct = (used: number | null | undefined, total: number | null | undefined) => (typeof used === "number" && typeof total === "number" && total > 0 ? Math.min(100, Math.max(0, (used / total) * 100)) : null);
|
||||
|
||||
/** Pure: the pools in what the health pass has just read. A node or controller that could not be read is not in the snapshot, so it adds nothing. */
|
||||
export function poolsOf(snapshot: Pick<HealthSnapshot, "proxmox" | "synology">): PoolReading[] {
|
||||
const out: PoolReading[] = [];
|
||||
for (const px of snapshot.proxmox) {
|
||||
const seenShared = new Set<string>();
|
||||
for (const node of px.nodes) {
|
||||
if (node.error) continue;
|
||||
const root = pct(node.rootfsUsedBytes, node.rootfsTotalBytes);
|
||||
if (root !== null) out.push({ integrationId: px.integrationId, key: `${node.node}:rootfs`, label: `${px.integrationName} / ${node.node}: root filesystem`, percent: root, usedBytes: node.rootfsUsedBytes, totalBytes: node.rootfsTotalBytes });
|
||||
for (const st of node.storages) {
|
||||
if (!st.active) continue;
|
||||
const p = pct(st.usedBytes, st.totalBytes);
|
||||
if (p === null) continue;
|
||||
if (st.shared) {
|
||||
if (seenShared.has(st.id)) continue;
|
||||
seenShared.add(st.id);
|
||||
out.push({ integrationId: px.integrationId, key: `shared:${st.id}`, label: `${px.integrationName}: shared storage "${st.id}"`, percent: p, usedBytes: st.usedBytes, totalBytes: st.totalBytes });
|
||||
} else {
|
||||
out.push({ integrationId: px.integrationId, key: `${node.node}:storage:${st.id}`, label: `${px.integrationName} / ${node.node}: storage "${st.id}"`, percent: p, usedBytes: st.usedBytes, totalBytes: st.totalBytes });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const syn of snapshot.synology) {
|
||||
for (const v of syn.storage.volumes) {
|
||||
const p = pct(v.sizeUsed, v.sizeTotal);
|
||||
if (p === null) continue;
|
||||
out.push({ integrationId: syn.integrationId, key: `volume:${v.id}`, label: `${syn.integrationName}: volume ${v.id}`, percent: p, usedBytes: v.sizeUsed, totalBytes: v.sizeTotal });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const round1 = (n: number) => Math.round(n * 10) / 10;
|
||||
|
||||
/** Writes down what this pass read. Returns how many samples were added. Never throws: the health pass must not fail for want of a history. */
|
||||
export async function recordStorageSamples(snapshot: Pick<HealthSnapshot, "proxmox" | "synology">, now: Date = new Date()): Promise<number> {
|
||||
let added = 0;
|
||||
try {
|
||||
const nowIso = now.toISOString();
|
||||
for (const r of poolsOf(snapshot)) {
|
||||
const values = { label: r.label, latestPercent: round1(r.percent), usedBytes: r.usedBytes === null ? null : Math.round(r.usedBytes), totalBytes: r.totalBytes === null ? null : Math.round(r.totalBytes), updatedAt: nowIso };
|
||||
const [series] = await db
|
||||
.insert(storageSeries)
|
||||
.values({ integrationId: r.integrationId, key: r.key, ...values })
|
||||
.onConflictDoUpdate({ target: [storageSeries.integrationId, storageSeries.key], set: values })
|
||||
.returning({ id: storageSeries.id });
|
||||
const [newest] = await db.select({ at: storageSamples.at }).from(storageSamples).where(eq(storageSamples.seriesId, series.id)).orderBy(desc(storageSamples.at)).limit(1);
|
||||
if (newest && now.getTime() - Date.parse(newest.at) < MIN_GAP_MS) continue;
|
||||
await db.insert(storageSamples).values({ seriesId: series.id, at: nowIso, value: round1(r.percent) });
|
||||
added++;
|
||||
}
|
||||
await db.delete(storageSamples).where(lt(storageSamples.at, new Date(now.getTime() - KEEP_DAYS * DAY_MS).toISOString()));
|
||||
await db.delete(storageSeries).where(lt(storageSeries.updatedAt, new Date(now.getTime() - FORGET_AFTER_DAYS * DAY_MS).toISOString()));
|
||||
} catch (err) {
|
||||
console.error("[storage] couldn't write down the storage figures:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
return added;
|
||||
}
|
||||
|
||||
export interface StorageTrendInfo {
|
||||
perDay: number;
|
||||
daysToFull: number | null;
|
||||
fit: number;
|
||||
spanDays: number;
|
||||
}
|
||||
|
||||
export interface StorageForecastRow {
|
||||
seriesId: number;
|
||||
integrationId: number;
|
||||
key: string;
|
||||
label: string;
|
||||
latest: number;
|
||||
usedBytes: number | null;
|
||||
totalBytes: number | null;
|
||||
updatedAt: string;
|
||||
/** The straight-line trend, when there is enough history for one; `steady` says whether it is one to believe. */
|
||||
trend: StorageTrendInfo | null;
|
||||
steady: boolean;
|
||||
}
|
||||
|
||||
/** How far back the longer look goes when the last two weeks say a pool is flat. */
|
||||
export const LONG_DAYS = 90;
|
||||
const MIN_SPAN_DAYS = 3;
|
||||
const MIN_FIT = 0.9;
|
||||
|
||||
const toInfo = (t: Trend): StorageTrendInfo => ({ perDay: t.perDay, daysToFull: t.daysToFull, fit: t.fit, spanDays: t.spanDays });
|
||||
|
||||
/** Every pool with its latest figure and trend: the last two weeks, or — when those are flat — the last ninety days if they are really rising. */
|
||||
export async function storageOverview(now: Date = new Date()): Promise<StorageForecastRow[]> {
|
||||
const series = await db.select().from(storageSeries);
|
||||
const since90 = new Date(now.getTime() - LONG_DAYS * DAY_MS).toISOString();
|
||||
const rows: StorageForecastRow[] = [];
|
||||
for (const s of series) {
|
||||
const samples = (await db.select({ at: storageSamples.at, value: storageSamples.value }).from(storageSamples).where(and(eq(storageSamples.seriesId, s.id), gte(storageSamples.at, since90))).orderBy(storageSamples.at)).map((r) => ({ t: Date.parse(r.at), v: r.value }));
|
||||
const cut14 = now.getTime() - 14 * DAY_MS;
|
||||
let trend = trendOf(samples.filter((p) => p.t >= cut14));
|
||||
if (!trend || trend.daysToFull === null) {
|
||||
const longer = trendOf(samples);
|
||||
if (longer && longer.daysToFull !== null) trend = longer;
|
||||
}
|
||||
rows.push({
|
||||
seriesId: s.id,
|
||||
integrationId: s.integrationId,
|
||||
key: s.key,
|
||||
label: s.label,
|
||||
latest: s.latestPercent,
|
||||
usedBytes: s.usedBytes,
|
||||
totalBytes: s.totalBytes,
|
||||
updatedAt: s.updatedAt,
|
||||
trend: trend ? toInfo(trend) : null,
|
||||
steady: !!trend && trend.spanDays >= MIN_SPAN_DAYS && trend.fit >= MIN_FIT,
|
||||
});
|
||||
}
|
||||
return rows.sort((a, b) => a.label.localeCompare(b.label));
|
||||
}
|
||||
|
||||
export interface StorageForecast {
|
||||
integrationId: number;
|
||||
key: string;
|
||||
label: string;
|
||||
latest: number;
|
||||
perDay: number;
|
||||
daysToFull: number;
|
||||
}
|
||||
|
||||
/** The pools heading for full within `withinDays` — the ones already past `limitPercent` are left to the ordinary "nearly full" alert. */
|
||||
export function storageFillingSoon(rows: StorageForecastRow[], withinDays: number, limitPercent: number): StorageForecast[] {
|
||||
const asSeries: MetricSeries[] = rows.map((r) => ({
|
||||
metric: `disk:${r.seriesId}`,
|
||||
kind: "disk",
|
||||
label: r.label,
|
||||
points: [],
|
||||
latest: r.latest,
|
||||
trend: r.trend ? { perDay: r.trend.perDay, fit: r.trend.fit, spanDays: r.trend.spanDays, daysToFull: r.trend.daysToFull } : null,
|
||||
}));
|
||||
const byId = new Map(rows.map((r) => [String(r.seriesId), r]));
|
||||
return disksFillingSoon(asSeries, withinDays, limitPercent).map((f) => {
|
||||
const r = byId.get(f.mount)!;
|
||||
return { integrationId: r.integrationId, key: r.key, label: r.label, latest: f.latest, perDay: f.perDay, daysToFull: f.daysToFull };
|
||||
});
|
||||
}
|
||||
|
||||
/** One pool's readings over the last `days`, for a chart. */
|
||||
export async function storageHistory(seriesId: number, days: number, now: Date = new Date()): Promise<{ t: string; v: number }[]> {
|
||||
const since = new Date(now.getTime() - Math.min(KEEP_DAYS, Math.max(1, days)) * DAY_MS).toISOString();
|
||||
return (await db.select({ t: storageSamples.at, v: storageSamples.value }).from(storageSamples).where(and(eq(storageSamples.seriesId, seriesId), gte(storageSamples.at, since))).orderBy(storageSamples.at));
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import { notify, notifyNow } from "./notify.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
||||
import { storageFillingSoon, storageOverview } from "./storageHistory.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
@@ -167,6 +168,11 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
|
||||
}
|
||||
}
|
||||
if (settings.healthChecks.diskForecastDays > 0) {
|
||||
for (const f of storageFillingSoon(await storageOverview(now), settings.healthChecks.diskForecastDays, diskUsagePercent)) {
|
||||
lines.push(`- ${f.label} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
|
||||
}
|
||||
}
|
||||
if (lines.length > 0) sections.push({ heading: "Disks heading for full", lines: listWithMore(lines), attention: true });
|
||||
}
|
||||
} catch (err) {
|
||||
|
||||
@@ -0,0 +1,325 @@
|
||||
/**
|
||||
* Alert rules of your own, forecasts for the pools the integrations report, and snoozing an alert.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-rss-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, serverMetrics, integrations, alertRules, storageSeries, storageSamples, alertSnoozes, maintenanceWindows } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { evaluateRules, currentRun, checkedRule, computeRuleFindings, listRules, findingKey, ruleCovers, RuleError } = await import("../src/services/alertRules.js");
|
||||
const { evaluateHealth } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { metaOfConditionKey } = await import("../src/services/notifyRouting.js");
|
||||
const { poolsOf, recordStorageSamples, storageOverview, storageFillingSoon } = await import("../src/services/storageHistory.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { setSnooze, clearSnooze, loadSnoozes, withoutSnoozed, settleSnoozes, SnoozeError } = await import("../src/services/alertSnoozes.js");
|
||||
const { alertRulesRouter } = await import("../src/routes/alertRules.js");
|
||||
const { alertsRouter } = await import("../src/routes/alerts.js");
|
||||
const { storageRouter } = await import("../src/routes/storage.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const DAY = 86_400_000;
|
||||
const NOW = Date.parse("2026-10-20T12:00:00Z");
|
||||
type Rule = Parameters<typeof evaluateRules>[0][number];
|
||||
const rule = (extra: Partial<Rule> = {}): Rule => ({ id: 1, name: "CPU hot", enabled: true, metric: "cpu", mount: null, operator: "above", threshold: 90, forMinutes: 15, severity: "warning", serverId: null, tag: null, createdBy: null, createdAt: "", ...extra });
|
||||
const samplesAt = (values: number[], endsAgoMin = 1, gapMin = 15) => values.map((v, i) => ({ t: NOW - (endsAgoMin + (values.length - 1 - i) * gapMin) * MIN, v }));
|
||||
const server = (id: number, name: string, tags: string[] = [], smart: { device: string; temperatureC: number | null }[] = []) => ({ id, name, tags, smart });
|
||||
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows may still hold the database file
|
||||
}
|
||||
});
|
||||
|
||||
describe("your own alert rules", () => {
|
||||
test("it has to have been past the limit for the whole time, on every sample, up to now", () => {
|
||||
assert.equal(currentRun(rule(), samplesAt([95]), NOW)?.since, NOW - 1 * MIN);
|
||||
const held = currentRun(rule(), samplesAt([95, 96, 97]), NOW)!;
|
||||
assert.deepEqual([held.value, (held.latest - held.since) / MIN], [97, 30]);
|
||||
assert.equal(currentRun(rule(), samplesAt([95, 60, 97]), NOW)!.since, NOW - 1 * MIN, "a dip starts the count again");
|
||||
assert.equal(currentRun(rule(), samplesAt([95, 96, 60]), NOW), null, "the newest sample is fine");
|
||||
assert.equal(currentRun(rule(), samplesAt([95, 96, 97], 50), NOW), null, "too old to say what it is now");
|
||||
assert.equal(currentRun(rule({ operator: "below", threshold: 10 }), samplesAt([5, 4]), NOW)!.value, 4);
|
||||
});
|
||||
|
||||
test("a rule with a time waits for it; without one it speaks at once; a different server or tag is left alone", () => {
|
||||
const s = [server(1, "web", ["prod"]), server(2, "pi", ["lab"])];
|
||||
const hot = new Map([["1|cpu", samplesAt([95, 96])], ["2|cpu", samplesAt([95, 96])]]);
|
||||
const found = (r: Rule, m = hot) => evaluateRules([r], s, m, NOW).map((f) => f.serverName);
|
||||
assert.deepEqual(found(rule()), ["web", "pi"], "30 minutes past the limit is more than the 15 asked");
|
||||
assert.deepEqual(found(rule({ forMinutes: 60 })), [], "but not an hour");
|
||||
assert.deepEqual(found(rule({ forMinutes: 0 }), new Map([["1|cpu", samplesAt([95])]])), ["web"], "one sample is enough without a time");
|
||||
assert.deepEqual(found(rule({ tag: "LAB" })), ["pi"], "a tag, however it is written");
|
||||
assert.deepEqual(found(rule({ serverId: 1 })), ["web"]);
|
||||
assert.equal(ruleCovers({ serverId: null, tag: null }, { id: 9, tags: [] }), true);
|
||||
assert.deepEqual(evaluateRules([rule({ enabled: false })], s, hot, NOW), []);
|
||||
});
|
||||
|
||||
test("a disk is judged mount by mount, or just the one named; a temperature is judged as of the last report, per disk", () => {
|
||||
const s = [server(1, "nas", [], [{ device: "/dev/sda", temperatureC: 58 }, { device: "/dev/sdb", temperatureC: 41 }, { device: "/dev/sdc", temperatureC: null }])];
|
||||
const disks = new Map([["1|disk:/", samplesAt([88, 92])], ["1|disk:/data", samplesAt([91, 93])], ["1|cpu", samplesAt([99])]]);
|
||||
const every = evaluateRules([rule({ metric: "disk", threshold: 90, forMinutes: 0 })], s, disks, NOW);
|
||||
assert.deepEqual(every.map((f) => f.detail), ["/", "/data"]);
|
||||
assert.match(every[1].message, /nas: \/data use is 93%, above 90% \[CPU hot\]/);
|
||||
assert.deepEqual(evaluateRules([rule({ metric: "disk", threshold: 90, forMinutes: 0, mount: "/data" })], s, disks, NOW).map((f) => f.detail), ["/data"]);
|
||||
const temp = evaluateRules([rule({ metric: "disk_temp", threshold: 55, forMinutes: 0, severity: "critical", name: "Warm disk" })], s, new Map(), NOW);
|
||||
assert.deepEqual(temp.map((f) => [f.detail, f.value, f.severity, f.since]), [["/dev/sda", 58, "critical", null]]);
|
||||
assert.match(temp[0].message, /nas: \/dev\/sda is 58 °C, above 55 °C \[Warm disk\]/);
|
||||
});
|
||||
|
||||
test("a finding is a health condition of the server — announced, silenced by maintenance and routed like the rest — unless the server is offline", () => {
|
||||
const f = evaluateRules([rule()], [server(1, "web")], new Map([["1|cpu", samplesAt([95, 96])]]), NOW);
|
||||
assert.equal(findingKey(f[0]), "rule:1:server:1");
|
||||
assert.equal(findingKey({ ruleId: 2, serverId: 4, detail: "/data" }), "rule:2:server:4:/data");
|
||||
const snap = (lastSeenAt: string) => ({ servers: [{ id: 1, name: "web", lastSeenAt, disks: [] }], proxmox: [], synology: [], rules: f });
|
||||
const thresholds = { serverOfflineMinutes: 60, diskUsagePercent: 90 };
|
||||
const ok = evaluateHealth(snap(new Date(NOW - MIN).toISOString()), thresholds, NOW);
|
||||
assert.deepEqual(ok.map((c) => [c.key, c.source, c.severity]), [["rule:1:server:1", "server:rules", "warning"]]);
|
||||
assert.equal(subjectOfConditionKey("rule:1:server:1"), "server:1");
|
||||
assert.equal(subjectOfConditionKey("rule:1:server:4:/data"), "server:4");
|
||||
assert.deepEqual(metaOfConditionKey("rule:1:server:4", "critical"), { category: "monitoring", severity: "critical", serverId: 4 });
|
||||
const down = evaluateHealth(snap(new Date(NOW - 3 * 3600_000).toISOString()), thresholds, NOW).map((c) => c.key);
|
||||
assert.deepEqual(down, ["offline:server:1"], "an offline server says so, and is not also judged on its load");
|
||||
assert.deepEqual(evaluateHealth(snap(new Date(NOW - MIN).toISOString()), thresholds, NOW, { skipServers: true }), []);
|
||||
});
|
||||
|
||||
test("what is asked for is checked, and said plainly when it is wrong", async () => {
|
||||
const bad = async (input: Parameters<typeof checkedRule>[0]) => checkedRule(input).then(() => null, (e) => (e instanceof RuleError ? e.code : "other"));
|
||||
assert.equal(await bad({ name: " ", metric: "cpu", threshold: 90 }), "invalid_name");
|
||||
assert.equal(await bad({ name: "x", metric: "cpu", threshold: 120 }), "invalid_threshold");
|
||||
assert.equal(await bad({ name: "x", metric: "disk_temp", threshold: 200 }), "invalid_threshold");
|
||||
assert.equal(await bad({ name: "x", metric: "cpu", threshold: 90, forMinutes: 5000 }), "invalid_for");
|
||||
assert.equal(await bad({ name: "x", metric: "cpu", threshold: 90, tag: "a b c d e f g h i j k l m n o p q r s t u v w x y z" }), "invalid_tag");
|
||||
assert.equal(await bad({ name: "x", metric: "cpu", threshold: 90, serverId: 999, }), "unknown_server");
|
||||
const ok = await checkedRule({ name: " Tidy ", metric: "disk_temp", threshold: 55, forMinutes: 30, mount: "/x", tag: " Prod " });
|
||||
assert.deepEqual([ok.name, ok.forMinutes, ok.mount, ok.tag], ["Tidy", 0, null, "prod"], "a temperature has no time and no mount; a tag is written the usual way");
|
||||
});
|
||||
});
|
||||
|
||||
describe("rules, the Alerts page and the pages that show them", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "viewer";
|
||||
let webId = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
const now = new Date();
|
||||
const [web] = await db.insert(servers).values({ name: "web", apiTokenHash: "h1", apiTokenPrefix: "p1", lastSeenAt: now.toISOString(), tags: JSON.stringify(["prod"]), ipAddresses: "[]" }).returning();
|
||||
webId = web.id;
|
||||
await db.insert(servers).values({ name: "pi", apiTokenHash: "h2", apiTokenPrefix: "p2", lastSeenAt: now.toISOString(), tags: JSON.stringify(["lab"]), ipAddresses: "[]" });
|
||||
// web has been at 95% for thirty minutes; pi is fine.
|
||||
for (const [id, vals] of [[web.id, [95, 96, 97]], [web.id + 1, [20, 25, 22]]] as const) {
|
||||
for (let i = 0; i < vals.length; i++) await db.insert(serverMetrics).values({ serverId: id, metric: "cpu", value: vals[i], recordedAt: new Date(now.getTime() - (vals.length - 1 - i) * 15 * MIN - MIN).toISOString() });
|
||||
}
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/alert-rules", alertRulesRouter);
|
||||
a.use("/api/alerts", alertsRouter);
|
||||
a.use("/api/storage", storageRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
const send = async (method: string, path: string, body?: unknown) => {
|
||||
const res = await fetch(`${base}${path}`, { method, headers: { "content-type": "application/json", connection: "close" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
return { status: res.status, body: res.status === 204 ? null : ((await res.json()) as any) };
|
||||
};
|
||||
const draft = { name: "Web CPU", metric: "cpu", threshold: 90, forMinutes: 15, severity: "critical" };
|
||||
|
||||
test("everyone reads the rules; only an administrator makes, tries, changes and removes them", async () => {
|
||||
who = "operator";
|
||||
assert.equal((await send("POST", "/alert-rules", draft)).status, 403);
|
||||
assert.equal((await send("POST", "/alert-rules/preview", draft)).status, 403);
|
||||
who = "admin";
|
||||
assert.equal((await send("POST", "/alert-rules", { ...draft, threshold: 500 })).body.error, "invalid_threshold");
|
||||
const preview = await send("POST", "/alert-rules/preview", draft);
|
||||
assert.deepEqual(preview.body.findings.map((f: any) => f.serverName), ["web"], "tried before it is saved");
|
||||
assert.equal((await db.select().from(alertRules)).length, 0, "and saving nothing");
|
||||
const made = await send("POST", "/alert-rules", draft);
|
||||
assert.equal(made.status, 201);
|
||||
who = "viewer";
|
||||
const list = await send("GET", "/alert-rules");
|
||||
assert.equal(list.status, 200);
|
||||
assert.deepEqual(list.body.rules.map((r: any) => [r.name, r.firing.map((f: any) => f.serverName)]), [["Web CPU", ["web"]]]);
|
||||
assert.deepEqual(list.body.tags, ["lab", "prod"]);
|
||||
assert.equal((await send("DELETE", `/alert-rules/${made.body.rule.id}`)).status, 403);
|
||||
});
|
||||
|
||||
test("a rule is a monitoring alert about the server, with its own severity, where to look, and who to silence it with", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const rid = (await listRules())[0].id;
|
||||
const alert = report.alerts.find((a) => a.id === `monitoring:rule:${rid}:server:${webId}`)!;
|
||||
assert.ok(alert, "raised");
|
||||
assert.deepEqual([alert.severity, alert.category, alert.source, alert.link, alert.target?.name, alert.silenced], ["critical", "monitoring", "Server", `/servers/${webId}?view=history`, "web", false]);
|
||||
assert.match(alert.message, /web: cpu load is 97%, above 90% for 15 min \[Web CPU\]/);
|
||||
assert.equal(report.alerts.some((a) => a.id.includes("server:" + (webId + 1)) && a.id.startsWith("monitoring:rule")), false, "pi is fine");
|
||||
const [win] = await db.insert(maintenanceWindows).values({ targetType: "server", targetId: webId, startedAt: new Date(Date.now() - 1000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() }).returning();
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await getAlerts(true)).alerts.find((a) => a.id === alert.id)!.silenced, true, "maintenance holds it");
|
||||
await db.delete(maintenanceWindows).where(eq(maintenanceWindows.id, win.id));
|
||||
});
|
||||
|
||||
test("switching it off, or changing the limit, takes the alert away; removing it too", async () => {
|
||||
who = "admin";
|
||||
const r = (await listRules())[0];
|
||||
assert.equal((await send("PUT", `/alert-rules/${r.id}`, { ...draft, enabled: false })).status, 200);
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await getAlerts(true)).alerts.some((a) => a.id.startsWith("monitoring:rule:")), false);
|
||||
assert.equal((await send("PUT", `/alert-rules/${r.id}`, { ...draft, threshold: 99 })).status, 200);
|
||||
assert.deepEqual((await computeRuleFindings(await listRules())), [], "99 is above all it has seen");
|
||||
assert.equal((await send("PUT", "/alert-rules/99999", draft)).status, 404);
|
||||
assert.equal((await send("DELETE", `/alert-rules/${r.id}`)).status, 204);
|
||||
assert.equal((await send("DELETE", `/alert-rules/${r.id}`)).status, 404);
|
||||
});
|
||||
|
||||
describe("storage forecasts", () => {
|
||||
const px = (usedGb: number, extra: { shared?: boolean; node?: string } = {}) => ({
|
||||
proxmox: [{ integrationId: 1, integrationName: "PVE", nodes: [{ node: extra.node ?? "pve1", error: null, uptime: 1, cpuUsagePercent: 1, cpuCores: 4, loadAverage: null, memTotalBytes: 1, memUsedBytes: 1, swapTotalBytes: null, swapUsedBytes: null, rootfsTotalBytes: 100e9, rootfsUsedBytes: 20e9, pveVersion: null, storages: [{ id: "tank", type: "zfs", active: true, shared: extra.shared ?? false, totalBytes: 1000e9, usedBytes: usedGb * 1e9, availBytes: null }, { id: "off", type: "dir", active: false, shared: false, totalBytes: 10, usedBytes: 5, availBytes: null }] }, { node: "pve2", error: "no access", uptime: null, cpuUsagePercent: null, cpuCores: null, loadAverage: null, memTotalBytes: null, memUsedBytes: null, swapTotalBytes: null, swapUsedBytes: null, rootfsTotalBytes: null, rootfsUsedBytes: null, pveVersion: null, storages: [] }] }],
|
||||
synology: [{ integrationId: 2, integrationName: "NAS", storage: { volumes: [{ id: "volume_1", status: "normal", deviceType: "raid", sizeTotal: 8e12, sizeUsed: 6e12 }, { id: "volume_2", status: "normal", deviceType: "raid", sizeTotal: null, sizeUsed: null }], disks: [] } }],
|
||||
});
|
||||
|
||||
test("the pools in a read: root filesystems, active storages (a shared one once), volumes; not a node that could not be read, an inactive storage or a volume with no size", () => {
|
||||
const pools = poolsOf(px(500));
|
||||
assert.deepEqual(pools.map((p) => [p.integrationId, p.key, Math.round(p.percent)]), [[1, "pve1:rootfs", 20], [1, "pve1:storage:tank", 50], [2, "volume:volume_1", 75]]);
|
||||
const shared = poolsOf({ ...px(500, { shared: true }), proxmox: [{ ...px(500, { shared: true }).proxmox[0], nodes: [px(500, { shared: true }).proxmox[0].nodes[0], { ...px(500, { shared: true }).proxmox[0].nodes[0], node: "pve3" }] }] });
|
||||
assert.equal(shared.filter((p) => p.key === "shared:tank").length, 1, "listed by every node, said once");
|
||||
});
|
||||
|
||||
test("a reading is kept an hour at most; the pool's latest figure is always the newest; a pool that is not reported for a month is forgotten", async () => {
|
||||
await db.insert(integrations).values([{ type: "proxmox", name: "PVE", baseUrl: "http://pve" }, { type: "synology", name: "NAS", baseUrl: "http://nas" }]);
|
||||
const t0 = new Date(NOW);
|
||||
assert.equal(await recordStorageSamples(px(500), t0), 3);
|
||||
assert.equal(await recordStorageSamples(px(505), new Date(NOW + 20 * MIN)), 0, "too soon");
|
||||
assert.equal(await recordStorageSamples(px(510), new Date(NOW + 70 * MIN)), 3);
|
||||
const series = await db.select().from(storageSeries);
|
||||
assert.equal(series.length, 3);
|
||||
assert.equal(series.find((s) => s.key === "pve1:storage:tank")!.latestPercent, 51, "the latest, from the reading that was kept");
|
||||
assert.equal((await db.select().from(storageSamples)).length, 6);
|
||||
await recordStorageSamples({ proxmox: [], synology: [] }, new Date(NOW + 40 * DAY));
|
||||
assert.equal((await db.select().from(storageSeries)).length, 0, "forgotten with its readings");
|
||||
assert.equal((await db.select().from(storageSamples)).length, 0);
|
||||
});
|
||||
|
||||
test("a pool on a steady climb is forecast; a flat one, a wobbly one and one already over the limit are not", async () => {
|
||||
const [pve] = await db.select().from(integrations).where(eq(integrations.type, "proxmox"));
|
||||
const make = async (key: string, label: string, valueAt: (day: number) => number, latest: number) => {
|
||||
const [s] = await db.insert(storageSeries).values({ integrationId: pve.id, key, label, latestPercent: latest, usedBytes: null, totalBytes: null, updatedAt: new Date(NOW).toISOString() }).returning();
|
||||
for (let d = 0; d <= 14; d += 0.5) await db.insert(storageSamples).values({ seriesId: s.id, at: new Date(NOW - (14 - d) * DAY).toISOString(), value: valueAt(d) });
|
||||
return s;
|
||||
};
|
||||
const climbing = await make("a:climb", "PVE / a: storage \"climb\"", (d) => 50 + d * 2.5, 85);
|
||||
await make("b:flat", "PVE / b: storage \"flat\"", () => 40, 40);
|
||||
await make("c:wobble", "PVE / c: storage \"wobble\"", (d) => 50 + (Math.floor(d * 2) % 2 === 0 ? 8 : -8), 58);
|
||||
await make("d:over", "PVE / d: storage \"over\"", (d) => 80 + d * 0.9, 92.6);
|
||||
const rows = await storageOverview(new Date(NOW));
|
||||
const by = Object.fromEntries(rows.map((r) => [r.key, r]));
|
||||
assert.equal(by["a:climb"].steady, true);
|
||||
assert.equal(by["b:flat"].trend?.daysToFull ?? null, null);
|
||||
assert.equal(by["c:wobble"].steady, false);
|
||||
const soon = storageFillingSoon(rows, 30, 90);
|
||||
assert.deepEqual(soon.map((f) => f.key), ["a:climb"], "over the limit, flat and erratic are left out");
|
||||
assert.equal(soon[0].daysToFull, Math.ceil((100 - 85) / 2.5));
|
||||
assert.equal(storageFillingSoon(rows, 5, 90).length, 0, "not within five days");
|
||||
|
||||
// On the Alerts page: raised, silenced with the integration, and gone when the forecast is turned off.
|
||||
invalidateAlertsCache();
|
||||
const alert = (await getAlerts(true)).alerts.find((a) => a.id === `disk:storage-forecast:${pve.id}:a:climb`)!;
|
||||
assert.ok(alert);
|
||||
assert.deepEqual([alert.severity, alert.source, alert.target?.type, alert.silenced], ["warning", "Proxmox", "integration", false]);
|
||||
assert.match(alert.message, /filling by about 2\.5 points a day — full in about 6 days \(85% used now\)/);
|
||||
who = "viewer";
|
||||
const api = await send("GET", "/storage");
|
||||
assert.equal(api.status, 200);
|
||||
assert.equal(api.body.pools.find((p: any) => p.key === "a:climb").steady, true);
|
||||
const hist = await send("GET", `/storage/${climbing.id}/history?days=30`);
|
||||
assert.ok(hist.body.points.length > 5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("snooze", () => {
|
||||
const inHours = (h: number) => new Date(Date.now() + h * 3600_000).toISOString();
|
||||
const alertIdOf = async (prefix: string) => (await getAlerts(true)).alerts.find((a) => a.id.startsWith(prefix))!;
|
||||
|
||||
test("only an operator snoozes; a time that has passed, or is a year off, or an alert that is not there, is refused", async () => {
|
||||
invalidateAlertsCache();
|
||||
const target = await alertIdOf("disk:storage-forecast");
|
||||
assert.ok(target, "something to snooze");
|
||||
who = "viewer";
|
||||
assert.equal((await send("PUT", "/alerts/snooze", { alertId: target.id, until: inHours(5) })).status, 403);
|
||||
who = "operator";
|
||||
assert.equal((await send("PUT", "/alerts/snooze", { alertId: target.id, until: inHours(-1) })).body.error, "invalid_until");
|
||||
assert.equal((await send("PUT", "/alerts/snooze", { alertId: target.id, until: inHours(24 * 400) })).body.error, "invalid_until");
|
||||
assert.equal((await send("PUT", "/alerts/snooze", { alertId: target.id, until: "tomorrow" })).status, 400);
|
||||
assert.equal((await send("PUT", "/alerts/snooze", { alertId: "disk:nothing", until: inHours(5) })).status, 404);
|
||||
});
|
||||
|
||||
test("a snoozed alert is held back like one under maintenance, with when it returns; waking it brings it back at once", async () => {
|
||||
who = "operator";
|
||||
const target = await alertIdOf("disk:storage-forecast");
|
||||
const until = inHours(18);
|
||||
const res = await send("PUT", "/alerts/snooze", { alertId: target.id, until });
|
||||
assert.equal(res.status, 200);
|
||||
const page = await send("GET", "/alerts");
|
||||
const row = page.body.alerts.find((a: any) => a.id === target.id);
|
||||
assert.deepEqual([row.silenced, row.snoozed.until, row.snoozed.by], [true, new Date(until).toISOString(), "Olga"]);
|
||||
assert.ok(page.body.counts.silenced >= 1, "counted with the held ones, not with those that want a look");
|
||||
assert.equal((await send("DELETE", `/alerts/snooze?alertId=${encodeURIComponent(target.id)}`)).status, 204);
|
||||
assert.equal((await send("DELETE", `/alerts/snooze?alertId=${encodeURIComponent(target.id)}`)).status, 404);
|
||||
const again = (await send("GET", "/alerts")).body.alerts.find((a: any) => a.id === target.id);
|
||||
assert.deepEqual([again.silenced, again.snoozed ?? null], [false, null]);
|
||||
});
|
||||
|
||||
test("messages about a snoozed alert are held; those about others are not; when the time is up a reminder is made for one that is still there, and nothing for one that went", async () => {
|
||||
const now = new Date();
|
||||
await setSnooze({ alertId: "disk:disk:server:1:/", conditionKey: "disk:server:1:/", message: "web: / is 95% full", until: new Date(now.getTime() + 2 * 3600_000), actor: "Olga" }, now);
|
||||
assert.deepEqual((await withoutSnoozed([{ key: "disk:server:1:/", message: "a" }, { key: "disk:server:2:/", message: "b" }, { message: "no key" }], now)).map((i) => i.message), ["b", "no key"]);
|
||||
assert.equal((await loadSnoozes(now)).has("disk:disk:server:1:/"), true);
|
||||
await assert.rejects(setSnooze({ alertId: "x", conditionKey: "x", message: "x", until: new Date(now.getTime() + 10_000), actor: "Olga" }, now), SnoozeError);
|
||||
|
||||
// Two hours and a bit later: it is still there, so it is a reminder; the snooze is gone, so the next message about it is heard.
|
||||
const later = new Date(now.getTime() + 2 * 3600_000 + MIN);
|
||||
await setSnooze({ alertId: "backup:gone", conditionKey: "gone", message: "gone soon", until: new Date(now.getTime() + 3 * 3600_000), actor: "Olga" }, now);
|
||||
const reminders = await settleSnoozes([{ id: "disk:disk:server:1:/", message: "web: / is 96% full", severity: "warning", key: "disk:server:1:/" }], later);
|
||||
assert.deepEqual(reminders, [{ key: "disk:server:1:/", message: "web: / is 96% full", severity: "warning" }]);
|
||||
assert.equal((await withoutSnoozed([{ key: "disk:server:1:/", message: "a" }], later)).length, 1);
|
||||
// One whose alert is gone, and has been for half an hour, is dropped quietly; the snooze ending for an alert that went is forgotten without a word.
|
||||
const muchLater = new Date(now.getTime() + 4 * 3600_000);
|
||||
assert.deepEqual(await settleSnoozes([], muchLater), []);
|
||||
assert.equal((await db.select().from(alertSnoozes)).length, 0);
|
||||
assert.equal(await clearSnooze("disk:disk:server:1:/"), null);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user