Notification routing rules, per-browser push wishes, escalation of unacknowledged alerts
- Routing: rules pick channels by kind, severity and server tags (first match wins, none = nobody); mixed batches split per route; recoveries follow their alert; quiet-hours digest keeps each notification's audience - Push: each subscribed browser can ask for a minimum severity and some kinds - Escalation: up to three timed steps re-notify alerts nobody has acknowledged, through chosen channels and an optional extra email; an acknowledgement stops them - Fix ntfy: titles with an em dash made every ntfy send fail (header not Latin-1); now RFC 2047 encoded Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9caa414829
commit
43fc5e0272
21 files changed
+4769
-55
No files matched your search
@@ -0,0 +1,5 @@
|
||||
ALTER TABLE `alert_incidents` ADD `escalated_step` integer DEFAULT 0 NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE `alert_incidents` ADD `escalated_at` text;--> statement-breakpoint
|
||||
ALTER TABLE `notification_queue` ADD `audience` text;--> statement-breakpoint
|
||||
ALTER TABLE `push_subscriptions` ADD `min_severity` text;--> statement-breakpoint
|
||||
ALTER TABLE `push_subscriptions` ADD `categories` text;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -260,6 +260,13 @@
|
||||
"when": 1792341456680,
|
||||
"tag": "0036_cheerful_slipstream",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 37,
|
||||
"version": "6",
|
||||
"when": 1792344782975,
|
||||
"tag": "0037_past_magik",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -64,6 +64,8 @@ export const notificationQueue = sqliteTable("notification_queue", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
title: text("title").notNull(),
|
||||
message: text("message").notNull(),
|
||||
/** JSON: who it is for (the channels and browsers a routing rule picked); null = everyone, as before routing existed. */
|
||||
audience: text("audience"),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
@@ -119,6 +121,10 @@ export const pushSubscriptions = sqliteTable("push_subscriptions", {
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
lastSuccessAt: text("last_success_at"),
|
||||
/** This browser only wants alerts at least this bad ("warning" or "critical"); null = every severity. */
|
||||
minSeverity: text("min_severity"),
|
||||
/** JSON string[] of the kinds of alert it wants; null = every kind. */
|
||||
categories: text("categories"),
|
||||
});
|
||||
|
||||
// "A needs B", for things that can be down: a server or an integration each. Polymorphic like maintenance_windows (no foreign key); an edge whose
|
||||
@@ -678,6 +684,8 @@ export const alertIncidents = sqliteTable(
|
||||
resolvedAt: text("resolved_at"), // ISO; null while it is open
|
||||
lastSeenAt: text("last_seen_at").notNull(), // ISO
|
||||
missingSince: text("missing_since"), // ISO; set while it has been absent from the last look(s), before it is called over
|
||||
escalatedStep: integer("escalated_step").notNull().default(0), // the highest escalation step already sent for this incident (0 = none)
|
||||
escalatedAt: text("escalated_at"), // ISO; when that was
|
||||
},
|
||||
(t) => [index("alert_incidents_key_idx").on(t.alertKey), index("alert_incidents_opened_idx").on(t.openedAt), index("alert_incidents_target_idx").on(t.targetType, t.targetId)],
|
||||
);
|
||||
|
||||
@@ -19,6 +19,7 @@ import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
|
||||
import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceScheduler.js";
|
||||
import { initMaintenanceScheduleScheduler } from "./services/maintenanceScheduleScheduler.js";
|
||||
import { initUpdateRuns } from "./services/updateRuns.js";
|
||||
import { initEscalation } from "./services/escalation.js";
|
||||
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
|
||||
import { initIncidentRecorder } from "./services/incidents.js";
|
||||
import { getAlerts } from "./services/alerts.js";
|
||||
@@ -57,6 +58,7 @@ await initMaintenanceScheduleScheduler();
|
||||
initUpdateRuns();
|
||||
await initSelfBackupScheduler();
|
||||
initIncidentRecorder(() => getAlerts(false));
|
||||
initEscalation();
|
||||
|
||||
createApp().listen(env.port, () => {
|
||||
console.log(`homelab-manager listening on port ${env.port}`);
|
||||
|
||||
@@ -7,6 +7,7 @@ import { requireAuth } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { checkEndpoint, getVapidKeys, sendPush } from "../services/webPush.js";
|
||||
import { CATEGORIES, parseCategories } from "../services/notifyRouting.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const pushRouter = Router();
|
||||
@@ -45,7 +46,7 @@ pushRouter.get("/mine", asyncHandler(async (req, res) => {
|
||||
const { webPush } = await getSettings();
|
||||
res.json({
|
||||
enabled: webPush.enabled,
|
||||
subscriptions: rows.map((r) => ({ id: r.id, label: r.label, host: hostOf(r.endpoint), endpoint: r.endpoint, createdAt: r.createdAt, lastSuccessAt: r.lastSuccessAt })),
|
||||
subscriptions: rows.map((r) => ({ id: r.id, label: r.label, host: hostOf(r.endpoint), endpoint: r.endpoint, createdAt: r.createdAt, lastSuccessAt: r.lastSuccessAt, minSeverity: r.minSeverity === "critical" || r.minSeverity === "warning" ? r.minSeverity : null, categories: parseCategories(r.categories) })),
|
||||
});
|
||||
}));
|
||||
|
||||
@@ -74,6 +75,25 @@ pushRouter.post("/subscribe", asyncHandler(async (req, res) => {
|
||||
res.status(201).json({ id: row.id });
|
||||
}));
|
||||
|
||||
// What a browser wants to be told: only the worse alerts, only some kinds. Its owner's to say (an administrator may too).
|
||||
const prefsSchema = z.object({
|
||||
minSeverity: z.enum(["warning", "critical"]).nullable(),
|
||||
categories: z.array(z.enum(CATEGORIES)).max(10).nullable(),
|
||||
});
|
||||
|
||||
pushRouter.patch("/:id", asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
const parsed = prefsSchema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Choose a severity (or any) and the kinds of alert (or all).", details: parsed.success ? undefined : parsed.error.flatten() });
|
||||
const me = req.currentUser!;
|
||||
const [row] = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.id, id)).limit(1);
|
||||
if (!row || (row.userId !== me.id && me.role !== "admin")) return res.status(404).json({ error: "not_found" });
|
||||
const categories = parsed.data.categories && parsed.data.categories.length > 0 ? parsed.data.categories : null;
|
||||
await db.update(pushSubscriptions).set({ minSeverity: parsed.data.minSeverity, categories: categories ? JSON.stringify(categories) : null }).where(eq(pushSubscriptions.id, id));
|
||||
await recordAudit({ actor: me, category: "push", action: "preferences", targetType: "push_subscription", targetId: id, detail: { host: hostOf(row.endpoint), label: row.label, minSeverity: parsed.data.minSeverity, categories } });
|
||||
res.json({ minSeverity: parsed.data.minSeverity, categories });
|
||||
}));
|
||||
|
||||
// Stops a browser being written to. A person removes their own; an administrator may remove anyone's.
|
||||
pushRouter.delete("/:id", asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
|
||||
@@ -15,6 +15,8 @@ import { scheduleQuietHoursFlush } from "../services/quietHoursScheduler.js";
|
||||
import { scheduleWeeklyDigest } from "../services/weeklyDigestScheduler.js";
|
||||
import { buildDigest, sendDigest } from "../services/weeklyDigest.js";
|
||||
import { enabledChannelCount } from "../services/notify.js";
|
||||
import { CATEGORIES, CHANNEL_IDS } from "../services/notifyRouting.js";
|
||||
import { InvalidTagError, normalizeTags } from "../services/serverTags.js";
|
||||
import { testGotify, testNtfy, testSmtp, testWebhook, flushQuietHoursQueue } from "../services/notify.js";
|
||||
import { listQueuedNotifications } from "../services/notificationQueue.js";
|
||||
import { buildExportPayload, encryptExport, decryptExport, applyImportPayload, type EncryptedExportFile } from "../services/configBackup.js";
|
||||
@@ -47,7 +49,51 @@ settingsRouter.get("/display", asyncHandler(async (_req, res) => {
|
||||
res.json({ display });
|
||||
}));
|
||||
|
||||
const channelEnum = z.enum(CHANNEL_IDS);
|
||||
|
||||
// Routing rules: which channels hear which notifications. A rule with no channels means "tell nobody" (the Alerts page still shows it).
|
||||
const routingSchema = z.object({
|
||||
rules: z
|
||||
.array(
|
||||
z.object({
|
||||
id: z.string().trim().min(1).max(40),
|
||||
name: z.string().trim().min(1).max(60),
|
||||
enabled: z.boolean(),
|
||||
categories: z.array(z.enum(CATEGORIES)).max(10),
|
||||
minSeverity: z.enum(["warning", "critical"]).nullable(),
|
||||
tags: z.array(z.string().trim().min(1).max(30)).max(10),
|
||||
channels: z.array(channelEnum).max(5),
|
||||
}),
|
||||
)
|
||||
.max(30),
|
||||
});
|
||||
|
||||
// Escalation: a few steps in time order. Each must say who to tell — a channel, or an address.
|
||||
const escalationSchema = z
|
||||
.object({
|
||||
enabled: z.boolean(),
|
||||
steps: z
|
||||
.array(
|
||||
z.object({
|
||||
afterMinutes: z.number().int().min(5).max(10080),
|
||||
minSeverity: z.enum(["warning", "critical"]),
|
||||
channels: z.array(channelEnum).max(5),
|
||||
extraEmail: z.string().trim().max(200).refine((v) => v === "" || /^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(v), "That isn't an email address"),
|
||||
evenInQuietHours: z.boolean(),
|
||||
}),
|
||||
)
|
||||
.max(3),
|
||||
})
|
||||
.superRefine((v, ctx) => {
|
||||
v.steps.forEach((s, i) => {
|
||||
if (s.channels.length === 0 && s.extraEmail === "") ctx.addIssue({ code: "custom", path: ["steps", i], message: "Choose a channel to tell, or an address to email." });
|
||||
if (i > 0 && s.afterMinutes <= v.steps[i - 1].afterMinutes) ctx.addIssue({ code: "custom", path: ["steps", i, "afterMinutes"], message: "Each step must come later than the one before." });
|
||||
});
|
||||
});
|
||||
|
||||
const updateSchema = z.object({
|
||||
routing: routingSchema.optional(),
|
||||
escalation: escalationSchema.optional(),
|
||||
gotify: z.object({ enabled: z.boolean(), url: z.string(), token: z.string(), priority: z.number() }).partial().optional(),
|
||||
ntfy: z.object({ enabled: z.boolean(), url: z.string(), topic: z.string(), token: z.string(), priority: z.number() }).partial().optional(),
|
||||
smtp: z
|
||||
@@ -126,6 +172,14 @@ settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
return res.status(400).json({ error: "invalid_body", details: parsed.error.flatten() });
|
||||
}
|
||||
|
||||
// Tags in routing rules are written the way server tags are, so they match.
|
||||
try {
|
||||
if (parsed.data.routing) for (const rule of parsed.data.routing.rules) rule.tags = normalizeTags(rule.tags);
|
||||
} catch (err) {
|
||||
if (err instanceof InvalidTagError) return res.status(400).json({ error: "invalid_tag", message: err.message });
|
||||
throw err;
|
||||
}
|
||||
|
||||
const before = await getSettings();
|
||||
const updated = await updateSettings(parsed.data);
|
||||
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* Escalation: an alert that has been raised for a while and that nobody has acknowledged is told again — through louder channels, and
|
||||
* perhaps to someone else. Steps are in order ("after 30 minutes: push; after 2 hours: email the neighbour who holds the key"), each is sent
|
||||
* at most once per incident (one occurrence of an alert; it starts over if the alert goes away and comes back), and an acknowledgement stops
|
||||
* the lot. Alerts that are silenced for maintenance, or held back because something they depend on is down, never escalate.
|
||||
*/
|
||||
import { and, eq, inArray, isNull } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { alertIncidents } from "../db/schema.js";
|
||||
import type { Alert } from "./alertTypes.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { loadAcks } from "./alertAcks.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { notifyAudience } from "./notify.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { severityRank, type EscalationStep } from "./notifyRouting.js";
|
||||
|
||||
export interface DueEscalation {
|
||||
alert: Pick<Alert, "id" | "severity" | "message" | "source">;
|
||||
incidentId: number;
|
||||
/** 1-based, in the order the steps are kept. */
|
||||
step: number;
|
||||
definition: EscalationStep;
|
||||
}
|
||||
|
||||
export interface OpenIncident {
|
||||
id: number;
|
||||
openedAt: string;
|
||||
escalatedStep: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Which escalations are due now. Pure. An alert gets at most one step per look — the first one that is due, applies to its severity and hasn't
|
||||
* been sent — so a long-neglected alert is told about once per look rather than all at once.
|
||||
*/
|
||||
export function dueEscalations(input: {
|
||||
alerts: Pick<Alert, "id" | "severity" | "silenced" | "message" | "source">[];
|
||||
acknowledged: Set<string>;
|
||||
incidents: Map<string, OpenIncident>;
|
||||
steps: EscalationStep[];
|
||||
now: Date;
|
||||
}): DueEscalation[] {
|
||||
const out: DueEscalation[] = [];
|
||||
for (const alert of input.alerts) {
|
||||
if (alert.silenced || alert.severity === "info" || input.acknowledged.has(alert.id)) continue;
|
||||
const incident = input.incidents.get(alert.id);
|
||||
if (!incident) continue;
|
||||
const ageMinutes = (input.now.getTime() - Date.parse(incident.openedAt)) / 60_000;
|
||||
for (let i = incident.escalatedStep; i < input.steps.length; i++) {
|
||||
const def = input.steps[i];
|
||||
if (ageMinutes < def.afterMinutes) break; // steps are in time order: none after this one is due either
|
||||
if (severityRank(alert.severity) < severityRank(def.minSeverity)) continue; // not bad enough for this step; a later one may be for it
|
||||
out.push({ alert, incidentId: incident.id, step: i + 1, definition: def });
|
||||
break;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const words = (minutes: number) => (minutes >= 120 && minutes % 60 === 0 ? `${minutes / 60} hours` : minutes === 60 ? "an hour" : `${minutes} minutes`);
|
||||
|
||||
/** One look: sends whatever is due. Returns how many alerts were escalated. */
|
||||
export async function runEscalation(now: Date = new Date()): Promise<number> {
|
||||
const { escalation } = await getSettings();
|
||||
if (!escalation.enabled || escalation.steps.length === 0) return 0;
|
||||
|
||||
const report = await getAlerts(false);
|
||||
const ids = report.alerts.map((a) => a.id);
|
||||
if (ids.length === 0) return 0;
|
||||
const open = await db.select().from(alertIncidents).where(and(isNull(alertIncidents.resolvedAt), inArray(alertIncidents.alertKey, ids)));
|
||||
const acks = await loadAcks(now);
|
||||
|
||||
const due = dueEscalations({
|
||||
alerts: report.alerts,
|
||||
acknowledged: new Set([...acks].filter(([, a]) => a.active).map(([id]) => id)),
|
||||
incidents: new Map(open.map((i) => [i.alertKey, { id: i.id, openedAt: i.openedAt, escalatedStep: i.escalatedStep }])),
|
||||
steps: escalation.steps,
|
||||
now,
|
||||
});
|
||||
|
||||
// One message for each step, listing the alerts it is for.
|
||||
const byStep = new Map<number, DueEscalation[]>();
|
||||
for (const d of due) byStep.set(d.step, [...(byStep.get(d.step) ?? []), d]);
|
||||
for (const [step, items] of byStep) {
|
||||
const def = items[0].definition;
|
||||
const lines = items.map((d) => `${d.alert.severity === "critical" ? "✕" : "⚠"} ${d.alert.message}`);
|
||||
const message = `Nobody has acknowledged ${items.length === 1 ? "this alert" : `these ${items.length} alerts`} after ${words(def.afterMinutes)}:\n\n${lines.join("\n")}\n\nAcknowledge ${items.length === 1 ? "it" : "them"} on the Alerts page to stop the reminders.`;
|
||||
try {
|
||||
await notifyAudience("Homelab Manager — Still not acknowledged", message, { channels: def.channels, pushIds: null, ...(def.extraEmail ? { email: def.extraEmail } : {}) }, { quietHours: !def.evenInQuietHours });
|
||||
} catch (err) {
|
||||
console.error("[escalation] couldn't send step", step, err);
|
||||
continue; // not marked as sent: the next look tries again
|
||||
}
|
||||
for (const d of items) {
|
||||
await db.update(alertIncidents).set({ escalatedStep: step, escalatedAt: now.toISOString() }).where(eq(alertIncidents.id, d.incidentId));
|
||||
await recordAudit({ category: "alert", action: "escalate", detail: { alertId: d.alert.id, message: d.alert.message, step, afterMinutes: def.afterMinutes, channels: def.channels, email: def.extraEmail || null } });
|
||||
}
|
||||
}
|
||||
return due.length;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
/** Looks every five minutes (a little after the incident recorder, which uses the same fresh result). */
|
||||
export function initEscalation(): void {
|
||||
if (timer) return;
|
||||
const tick = () =>
|
||||
void runEscalation().catch((err) => console.error("[escalation] the look failed:", err));
|
||||
setTimeout(tick, 120_000).unref();
|
||||
timer = setInterval(tick, 5 * 60_000);
|
||||
timer.unref();
|
||||
}
|
||||
@@ -224,7 +224,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
// ─── State diff (pure) ──────────────────────────────────────────────────────
|
||||
|
||||
/** `covered`: the problem is explained by another that is being reported (a guest down because its host is) — recorded, but not announced on its own. */
|
||||
export type ActiveState = Record<string, { message: string; source: string; covered?: boolean }>;
|
||||
export type ActiveState = Record<string, { message: string; source: string; covered?: boolean; severity?: "critical" | "warning" }>;
|
||||
|
||||
/** `held` is a set of hierarchical sources that couldn't be read this run; matches the source itself or anything beneath it. */
|
||||
function isHeld(source: string, held: Set<string>): boolean {
|
||||
@@ -252,7 +252,7 @@ export function diffConditions(
|
||||
covered: Set<string> = new Set(),
|
||||
): {
|
||||
added: HealthCondition[];
|
||||
resolved: { key: string; message: string }[];
|
||||
resolved: { key: string; message: string; severity?: "critical" | "warning" }[];
|
||||
next: ActiveState;
|
||||
/** Every problem that began, announced or not. */
|
||||
newlyDown: HealthCondition[];
|
||||
@@ -268,12 +268,12 @@ export function diffConditions(
|
||||
const visible = current.filter((c) => !isSilenced(c.key));
|
||||
|
||||
const next: ActiveState = {};
|
||||
for (const c of visible) next[c.key] = covered.has(c.key) ? { message: c.message, source: c.source, covered: true } : { message: c.message, source: c.source };
|
||||
for (const c of visible) next[c.key] = { message: c.message, source: c.source, ...(covered.has(c.key) ? { covered: true } : {}), ...(c.severity ? { severity: c.severity } : {}) };
|
||||
|
||||
const newlyDown = visible.filter((c) => !(c.key in previous));
|
||||
// Announced: new and standing alone, or covered until now and no longer.
|
||||
const added = visible.filter((c) => !covered.has(c.key) && (!(c.key in previous) || previous[c.key].covered));
|
||||
const resolved: { key: string; message: string }[] = [];
|
||||
const resolved: { key: string; message: string; severity?: "critical" | "warning" }[] = [];
|
||||
const ended: { key: string; message: string }[] = [];
|
||||
for (const [key, entry] of Object.entries(previous)) {
|
||||
if (key in next) continue;
|
||||
@@ -282,7 +282,7 @@ export function diffConditions(
|
||||
next[key] = entry;
|
||||
} else {
|
||||
ended.push({ key, message: entry.message });
|
||||
if (!entry.covered) resolved.push({ key, message: entry.message });
|
||||
if (!entry.covered) resolved.push({ key, message: entry.message, ...(entry.severity ? { severity: entry.severity } : {}) });
|
||||
}
|
||||
}
|
||||
return { added, resolved, next, newlyDown, ended };
|
||||
|
||||
@@ -5,11 +5,13 @@ export interface QueuedNotification {
|
||||
id: number;
|
||||
title: string;
|
||||
message: string;
|
||||
/** JSON of who it was for; null means everyone. */
|
||||
audience: string | null;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export async function enqueueNotification(title: string, message: string): Promise<void> {
|
||||
await db.insert(notificationQueue).values({ title, message });
|
||||
export async function enqueueNotification(title: string, message: string, audience: string | null = null): Promise<void> {
|
||||
await db.insert(notificationQueue).values({ title, message, audience });
|
||||
}
|
||||
|
||||
export async function listQueuedNotifications(): Promise<QueuedNotification[]> {
|
||||
|
||||
+170
-38
@@ -3,6 +3,11 @@ import { getSettings, type NotificationEvents } from "./settingsStore.js";
|
||||
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
|
||||
import { emitEvent } from "./eventWebhooks.js";
|
||||
import { sendPush, subscriptionCount } from "./webPush.js";
|
||||
import { db } from "../db/client.js";
|
||||
import { inArray } from "drizzle-orm";
|
||||
import { pushSubscriptions, servers } from "../db/schema.js";
|
||||
import { parseTags } from "./serverTags.js";
|
||||
import { firstMatchingRule, metaOfConditionKey, pushAccepts, type ChannelId, type NotifyMeta } from "./notifyRouting.js";
|
||||
|
||||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||||
|
||||
@@ -22,13 +27,22 @@ async function sendGotify(title: string, message: string) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A header can only carry Latin-1, and every title here has an em dash. ntfy reads RFC 2047 encoded words in its headers ("=?UTF-8?B?...?="),
|
||||
* so anything outside plain ASCII is sent that way.
|
||||
*/
|
||||
export function headerSafe(text: string): string {
|
||||
// eslint-disable-next-line no-control-regex
|
||||
return /^[ -~]*$/.test(text) ? text : `=?UTF-8?B?${Buffer.from(text, "utf8").toString("base64")}?=`;
|
||||
}
|
||||
|
||||
async function sendNtfy(title: string, message: string) {
|
||||
const { ntfy } = await getSettings();
|
||||
if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return;
|
||||
const base = ntfy.url.replace(/\/$/, "");
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "text/plain",
|
||||
Title: title,
|
||||
Title: headerSafe(title),
|
||||
Priority: String(ntfy.priority ?? 3),
|
||||
};
|
||||
if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`;
|
||||
@@ -73,18 +87,116 @@ async function sendWebhook(title: string, message: string) {
|
||||
}
|
||||
}
|
||||
|
||||
async function sendWebPush(title: string, message: string) {
|
||||
async function sendWebPush(title: string, message: string, only: number[] | null) {
|
||||
const { webPush } = await getSettings();
|
||||
if (!webPush.enabled) return;
|
||||
try {
|
||||
await sendPush({ title, body: message });
|
||||
if (only === null) await sendPush({ title, body: message });
|
||||
else if (only.length > 0) await sendPush({ title, body: message }, await db.select().from(pushSubscriptions).where(inArray(pushSubscriptions.id, only)));
|
||||
} catch (err) {
|
||||
console.error("[notify] Web push failed:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
}
|
||||
|
||||
async function sendToChannels(title: string, message: string): Promise<void> {
|
||||
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message), sendWebPush(title, message)]);
|
||||
/** Email to one address, through the email channel's server, whatever the channel's own recipient is (an escalation to someone else). */
|
||||
async function sendExtraEmail(title: string, message: string, to: string) {
|
||||
const { smtp } = await getSettings();
|
||||
if (!smtp.host || !smtp.from) {
|
||||
console.error("[notify] An escalation wants to email someone, but email isn't set up.");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: smtp.host,
|
||||
port: Number(smtp.port) || 587,
|
||||
secure: !!smtp.secure,
|
||||
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
|
||||
});
|
||||
await transporter.sendMail({ from: smtp.from, to, subject: title, text: message });
|
||||
} catch (err) {
|
||||
console.error("[notify] Extra email failed:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Who a notification is for ─────────────────────────────────────────────
|
||||
|
||||
/** The channels and browsers a notification goes to. `null` means every one that is on (the way it was before routing existed). */
|
||||
export interface Audience {
|
||||
channels: ChannelId[] | null;
|
||||
/** The subscribed browsers that want it; null = all of them. */
|
||||
pushIds: number[] | null;
|
||||
/** Also email this address. */
|
||||
email?: string;
|
||||
}
|
||||
|
||||
export const EVERYONE: Audience = { channels: null, pushIds: null };
|
||||
|
||||
export const audienceKey = (a: Audience) => JSON.stringify({ c: a.channels ? [...a.channels].sort() : null, p: a.pushIds ? [...a.pushIds].sort((x, y) => x - y) : null, e: a.email ?? null });
|
||||
|
||||
/** Told to nobody: a rule said so (the Alerts page still shows it). */
|
||||
const isMuted = (a: Audience) => a.channels !== null && a.channels.length === 0 && !a.email;
|
||||
|
||||
/** Works out who a notification is for: the first routing rule it matches picks the channels, and browsers may turn down what they don't want. */
|
||||
export async function audienceFor(meta?: NotifyMeta): Promise<Audience> {
|
||||
if (!meta) return EVERYONE;
|
||||
const { routing } = await getSettings();
|
||||
let channels: ChannelId[] | null = null;
|
||||
if (routing.rules.some((r) => r.enabled)) {
|
||||
let tags = meta.tags;
|
||||
if (!tags && meta.serverId != null) {
|
||||
const [row] = await db.select({ tags: servers.tags }).from(servers).where(inArray(servers.id, [meta.serverId]));
|
||||
tags = parseTags(row?.tags);
|
||||
}
|
||||
channels = firstMatchingRule(routing.rules, { ...meta, tags: tags ?? [] })?.channels ?? null;
|
||||
}
|
||||
let pushIds: number[] | null = null;
|
||||
if (channels === null || channels.includes("webPush")) {
|
||||
const subs = await db.select({ id: pushSubscriptions.id, minSeverity: pushSubscriptions.minSeverity, categories: pushSubscriptions.categories }).from(pushSubscriptions);
|
||||
// Only when somebody has said what they want; otherwise everyone gets it, as ever.
|
||||
if (subs.some((s) => s.minSeverity || s.categories)) pushIds = subs.filter((s) => pushAccepts(s, meta)).map((s) => s.id);
|
||||
}
|
||||
return { channels, pushIds };
|
||||
}
|
||||
|
||||
async function sendToChannels(title: string, message: string, audience: Audience = EVERYONE): Promise<void> {
|
||||
const to = (c: ChannelId) => audience.channels === null || audience.channels.includes(c);
|
||||
await Promise.all([
|
||||
to("gotify") ? sendGotify(title, message) : undefined,
|
||||
to("ntfy") ? sendNtfy(title, message) : undefined,
|
||||
to("smtp") ? sendSmtp(title, message) : undefined,
|
||||
to("webhook") ? sendWebhook(title, message) : undefined,
|
||||
to("webPush") ? sendWebPush(title, message, audience.pushIds) : undefined,
|
||||
audience.email ? sendExtraEmail(title, message, audience.email) : undefined,
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends to an audience that is already worked out: held for the quiet-hours digest if it is quiet (unless told not to), nowhere if a rule
|
||||
* said nobody. Escalations use this with their own channels.
|
||||
*/
|
||||
export async function notifyAudience(title: string, message: string, audience: Audience, options: { quietHours?: boolean } = {}): Promise<void> {
|
||||
if (isMuted(audience)) return;
|
||||
if (options.quietHours !== false && (await isQuietHoursActive())) {
|
||||
await enqueueNotification(title, message, audience.channels === null && audience.pushIds === null && !audience.email ? null : JSON.stringify(audience));
|
||||
return;
|
||||
}
|
||||
await sendToChannels(title, message, audience);
|
||||
}
|
||||
|
||||
/** Items that may each be for a different audience (a rule can send a disk warning one place and a server outage another): sent as one message per audience. */
|
||||
async function notifyGrouped<T>(items: T[], metaOf: (item: T) => NotifyMeta, build: (group: T[]) => { title: string; message: string }): Promise<void> {
|
||||
const groups = new Map<string, { audience: Audience; items: T[] }>();
|
||||
for (const item of items) {
|
||||
const audience = await audienceFor(metaOf(item));
|
||||
const key = audienceKey(audience);
|
||||
const group = groups.get(key) ?? { audience, items: [] };
|
||||
group.items.push(item);
|
||||
groups.set(key, group);
|
||||
}
|
||||
for (const group of groups.values()) {
|
||||
const { title, message } = build(group.items);
|
||||
await notifyAudience(title, message, group.audience);
|
||||
}
|
||||
}
|
||||
|
||||
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
|
||||
@@ -133,23 +245,32 @@ export async function enabledChannelCount(): Promise<number> {
|
||||
].filter(Boolean).length;
|
||||
}
|
||||
|
||||
export async function notify(title: string, message: string): Promise<void> {
|
||||
if (await isQuietHoursActive()) {
|
||||
await enqueueNotification(title, message);
|
||||
return;
|
||||
}
|
||||
await sendToChannels(title, message);
|
||||
/** `meta` says what it is about, so routing rules and browsers' wishes can apply; without it, it goes to everyone. */
|
||||
export async function notify(title: string, message: string, meta?: NotifyMeta): Promise<void> {
|
||||
await notifyAudience(title, message, await audienceFor(meta));
|
||||
}
|
||||
|
||||
/** Called by the daily scheduler at quiet-hours end (or manually) — sends everything queued during the window as one digest, then clears it. */
|
||||
export async function flushQuietHoursQueue(): Promise<number> {
|
||||
const queued = await listQueuedNotifications();
|
||||
if (queued.length === 0) return 0;
|
||||
const sections = queued.map((q) => `— ${q.title} —\n${q.message}`);
|
||||
await sendToChannels(
|
||||
"Homelab Manager — Notifications from quiet hours",
|
||||
`${queued.length} notification${queued.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
|
||||
);
|
||||
// One digest for each audience the held notifications were for.
|
||||
const groups = new Map<string, typeof queued>();
|
||||
for (const q of queued) groups.set(q.audience ?? "", [...(groups.get(q.audience ?? "") ?? []), q]);
|
||||
for (const [stored, items] of groups) {
|
||||
let audience: Audience = EVERYONE;
|
||||
try {
|
||||
if (stored) audience = JSON.parse(stored) as Audience;
|
||||
} catch {
|
||||
// unreadable: everyone, as before
|
||||
}
|
||||
const sections = items.map((q) => `— ${q.title} —\n${q.message}`);
|
||||
await sendToChannels(
|
||||
"Homelab Manager — Notifications from quiet hours",
|
||||
`${items.length} notification${items.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
|
||||
audience,
|
||||
);
|
||||
}
|
||||
await clearQueuedNotifications();
|
||||
return queued.length;
|
||||
}
|
||||
@@ -172,7 +293,7 @@ export async function testGotify(cfg: { url: string; token: string; priority?: n
|
||||
|
||||
export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise<void> {
|
||||
const base = cfg.url.replace(/\/$/, "");
|
||||
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: "Homelab Manager — Test", Priority: String(cfg.priority ?? 3) };
|
||||
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: headerSafe("Homelab Manager — Test"), Priority: String(cfg.priority ?? 3) };
|
||||
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
|
||||
const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, {
|
||||
method: "POST",
|
||||
@@ -263,7 +384,7 @@ export async function notifySecretExpiry(
|
||||
`Couldn't read the live certificate for ${checkFailures.length} monitored secret${checkFailures.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
|
||||
);
|
||||
}
|
||||
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"));
|
||||
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"), { category: "expiry", severity: expiring.some((s) => s.status === "expired") ? "critical" : "warning" });
|
||||
}
|
||||
|
||||
export async function notifyDockerUpdates(
|
||||
@@ -277,6 +398,7 @@ export async function notifyDockerUpdates(
|
||||
await notify(
|
||||
"Homelab Manager — Docker Updates Available",
|
||||
`${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`,
|
||||
{ category: "updates", severity: "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -291,6 +413,7 @@ export async function notifyProxmoxBackupFailure(
|
||||
await notify(
|
||||
"Homelab Manager — Proxmox Backup Failed",
|
||||
`${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
|
||||
{ category: "backup", severity: "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -303,6 +426,7 @@ export async function notifyProxmoxUncoveredGuests(
|
||||
await notify(
|
||||
"Homelab Manager — Proxmox Guests Without Backup Coverage",
|
||||
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
|
||||
{ category: "backup", severity: "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -312,6 +436,7 @@ export async function notifyStaleBackups(stale: { message: string }[]): Promise<
|
||||
await notify(
|
||||
"Homelab Manager — Backups Gone Stale",
|
||||
`${stale.length} guest${stale.length !== 1 ? "s have" : " has"} no recent backup:\n\n${stale.map((s) => s.message).join("\n")}`,
|
||||
{ category: "backup", severity: "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -320,6 +445,7 @@ export async function notifyUpdateRunFinished(run: { serverName: string; templat
|
||||
await notify(
|
||||
run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`,
|
||||
`"${run.templateName}" on ${run.serverName}: ${run.message}`,
|
||||
{ category: "updates", severity: run.ok ? "warning" : "critical" },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -336,28 +462,32 @@ export async function notifyPbsVerificationFailed(
|
||||
await notify(
|
||||
"Homelab Manager — Proxmox Backup Server Verification Failed",
|
||||
`${failures.length} datastore${failures.length !== 1 ? "s have" : " has"} a problem:\n\n${lines.join("\n")}`,
|
||||
{ category: "backup", severity: "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
/** What a problem is about, for routing: its key says the kind and the server; the rest follows its severity. */
|
||||
const issueMeta = (i: { key?: string; severity?: string }): NotifyMeta => (i.key ? metaOfConditionKey(i.key, i.severity) : { category: null, severity: i.severity === "critical" ? "critical" : "warning" });
|
||||
|
||||
export async function notifyHealthIssues(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
|
||||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||||
if (issues.length === 0) return;
|
||||
if (!(await eventEnabled("healthAlerts"))) return;
|
||||
await notify(
|
||||
"Homelab Manager — Health Alert",
|
||||
`${issues.length} new issue${issues.length !== 1 ? "s" : ""}:\n\n${issues.map((i) => `⚠ ${i.message}`).join("\n")}`,
|
||||
);
|
||||
await notifyGrouped(issues, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Health Alert",
|
||||
message: `${group.length} new issue${group.length !== 1 ? "s" : ""}:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function notifyHealthRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
|
||||
export async function notifyHealthRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||||
if (resolved.length === 0) return;
|
||||
if (!(await eventEnabled("healthAlerts"))) return;
|
||||
await notify(
|
||||
"Homelab Manager — Health Recovered",
|
||||
`${resolved.length} issue${resolved.length !== 1 ? "s" : ""} cleared:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||||
);
|
||||
await notifyGrouped(resolved, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Health Recovered",
|
||||
message: `${group.length} issue${group.length !== 1 ? "s" : ""} cleared:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
|
||||
@@ -365,20 +495,20 @@ export async function notifyAutomationFailed(issues: { message: string; key?: st
|
||||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||||
if (issues.length === 0) return;
|
||||
if (!(await eventEnabled("automationAlerts"))) return;
|
||||
await notify(
|
||||
"Homelab Manager — Automation Failed",
|
||||
`${issues.length} run${issues.length !== 1 ? "s" : ""} failed:\n\n${issues.map((i) => `✕ ${i.message}`).join("\n")}`,
|
||||
);
|
||||
await notifyGrouped(issues, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Automation Failed",
|
||||
message: `${group.length} run${group.length !== 1 ? "s" : ""} failed:\n\n${group.map((i) => `✕ ${i.message}`).join("\n")}`,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function notifyAutomationRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
|
||||
export async function notifyAutomationRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||||
if (resolved.length === 0) return;
|
||||
if (!(await eventEnabled("automationAlerts"))) return;
|
||||
await notify(
|
||||
"Homelab Manager — Automation Recovered",
|
||||
`${resolved.length} no longer failing:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||||
);
|
||||
await notifyGrouped(resolved, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Automation Recovered",
|
||||
message: `${group.length} no longer failing:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function notifyDomainExpiry(
|
||||
@@ -403,7 +533,7 @@ export async function notifyDomainExpiry(
|
||||
`Couldn't refresh the registration for ${staleChecks.length} domain${staleChecks.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
|
||||
);
|
||||
}
|
||||
await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"));
|
||||
await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"), { category: "expiry", severity: expiring.some((d) => d.status === "expired") ? "critical" : "warning" });
|
||||
}
|
||||
|
||||
export async function notifyTailscaleKeyExpiry(
|
||||
@@ -417,6 +547,7 @@ export async function notifyTailscaleKeyExpiry(
|
||||
await notify(
|
||||
"Homelab Manager — Tailscale Key Expiry",
|
||||
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
|
||||
{ category: "expiry", severity: expiring.some((d) => d.daysLeft < 0) ? "critical" : "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -450,11 +581,12 @@ export async function notifyIntegrationDown(source: string, consecutiveFailures:
|
||||
await notify(
|
||||
"Homelab Manager — Integration Down",
|
||||
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
|
||||
{ category: "integration", severity: "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyIntegrationRecovered(source: string): Promise<void> {
|
||||
emitEvent("alert.resolved", { alert: { key: `integration:${source}`, message: `${sourceLabel(source)} is answering again.` } });
|
||||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||||
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
|
||||
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`, { category: "integration", severity: "warning" });
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* Who hears what. By default every notification goes to every channel that is switched on. Routing rules narrow that: the first rule a
|
||||
* notification matches decides which channels get it (possibly none — it is then only on the Alerts page), and each browser that has asked
|
||||
* for push notifications may further say it only wants some severities or kinds. This file is the pure part: what a notification is about,
|
||||
* which rule matches and what a browser accepts. Sending is in notify.ts.
|
||||
*/
|
||||
import type { AlertCategory, AlertSeverity } from "./alertTypes.js";
|
||||
|
||||
export const CHANNEL_IDS = ["gotify", "ntfy", "smtp", "webhook", "webPush"] as const;
|
||||
export type ChannelId = (typeof CHANNEL_IDS)[number];
|
||||
|
||||
export const CATEGORIES = ["offline", "disk", "services", "backup", "updates", "expiry", "automation", "integration", "monitoring", "tickets"] as const satisfies readonly AlertCategory[];
|
||||
|
||||
/** What a notification is about, as far as routing cares. Anything left out matches only rules that don't ask about it. */
|
||||
export interface NotifyMeta {
|
||||
category?: AlertCategory | null;
|
||||
severity?: AlertSeverity;
|
||||
/** The server it is about, whose tags a rule may ask for. */
|
||||
serverId?: number | null;
|
||||
/** Filled in from the server before matching. */
|
||||
tags?: string[];
|
||||
}
|
||||
|
||||
export interface RoutingRule {
|
||||
id: string;
|
||||
name: string;
|
||||
enabled: boolean;
|
||||
/** Empty = any kind. */
|
||||
categories: AlertCategory[];
|
||||
/** null = any severity. */
|
||||
minSeverity: "warning" | "critical" | null;
|
||||
/** Empty = any server. Otherwise the notification must be about a server that has at least one of these tags. */
|
||||
tags: string[];
|
||||
/** The channels that get it. Empty = none (the Alerts page still shows it). */
|
||||
channels: ChannelId[];
|
||||
}
|
||||
|
||||
export interface RoutingSettings {
|
||||
rules: RoutingRule[];
|
||||
}
|
||||
|
||||
export interface EscalationStep {
|
||||
/** Minutes an alert must have been raised, with nobody on it, before this step fires. */
|
||||
afterMinutes: number;
|
||||
minSeverity: "warning" | "critical";
|
||||
channels: ChannelId[];
|
||||
/** An address to email as well (someone else), through the email channel's server. Blank = none. */
|
||||
extraEmail: string;
|
||||
/** Send even during quiet hours — the point of an escalation can be to wake someone. */
|
||||
evenInQuietHours: boolean;
|
||||
}
|
||||
|
||||
export interface EscalationSettings {
|
||||
enabled: boolean;
|
||||
steps: EscalationStep[];
|
||||
}
|
||||
|
||||
const RANK: Record<string, number> = { info: 0, warning: 1, critical: 2 };
|
||||
export const severityRank = (s: string | undefined | null) => RANK[s ?? "warning"] ?? 1;
|
||||
|
||||
/** Whether a rule applies to a notification. A rule that asks about something the notification doesn't say doesn't match. */
|
||||
export function ruleMatches(rule: RoutingRule, meta: NotifyMeta): boolean {
|
||||
if (!rule.enabled) return false;
|
||||
if (rule.categories.length > 0 && !(meta.category && rule.categories.includes(meta.category))) return false;
|
||||
if (rule.minSeverity && severityRank(meta.severity) < severityRank(rule.minSeverity)) return false;
|
||||
if (rule.tags.length > 0) {
|
||||
const have = new Set((meta.tags ?? []).map((t) => t.toLowerCase()));
|
||||
if (!rule.tags.some((t) => have.has(t.toLowerCase()))) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** The first rule that applies, or null (every channel that is on). */
|
||||
export function firstMatchingRule(rules: RoutingRule[], meta: NotifyMeta): RoutingRule | null {
|
||||
return rules.find((r) => ruleMatches(r, meta)) ?? null;
|
||||
}
|
||||
|
||||
/** A browser's own wishes, as stored. */
|
||||
export interface PushPrefs {
|
||||
minSeverity: string | null;
|
||||
categories: string | null;
|
||||
}
|
||||
|
||||
export function parseCategories(stored: string | null): AlertCategory[] | null {
|
||||
if (!stored) return null;
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
if (!Array.isArray(v)) return null;
|
||||
const list = v.filter((c): c is AlertCategory => (CATEGORIES as readonly string[]).includes(c));
|
||||
return list.length > 0 ? list : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether a browser wants this notification. What the notification doesn't say can't exclude it. */
|
||||
export function pushAccepts(prefs: PushPrefs, meta: NotifyMeta | undefined): boolean {
|
||||
if (!meta) return true;
|
||||
if (prefs.minSeverity && meta.severity && severityRank(meta.severity) < severityRank(prefs.minSeverity)) return false;
|
||||
const wanted = parseCategories(prefs.categories);
|
||||
if (wanted && meta.category && !wanted.includes(meta.category)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/** What kind of problem a health or automation condition key is about, and which server (for its tags). */
|
||||
export function metaOfConditionKey(key: string, severity?: string): NotifyMeta {
|
||||
let category: AlertCategory | null = null;
|
||||
if (key.startsWith("offline:")) category = "offline";
|
||||
else if (key.startsWith("failed-units:")) category = "services";
|
||||
else if (/^(disk|smart|zfs|synology-volume|synology-disk):/.test(key)) category = "disk";
|
||||
else if (key.startsWith("automation:")) category = "automation";
|
||||
const server = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)/.exec(key);
|
||||
return { category, severity: severity === "critical" ? "critical" : "warning", serverId: server ? Number(server[1]) : null };
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { sql } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { settings } from "../db/schema.js";
|
||||
import { defaultThemes, type NameTheme } from "./nameThemes.js";
|
||||
import type { EscalationSettings, RoutingSettings } from "./notifyRouting.js";
|
||||
import { env } from "../env.js";
|
||||
import { decryptSecret, encryptSecret } from "../crypto.js";
|
||||
|
||||
@@ -44,6 +45,8 @@ export interface WebPushSettings {
|
||||
contact: string;
|
||||
}
|
||||
|
||||
export type { RoutingSettings, EscalationSettings } from "./notifyRouting.js";
|
||||
|
||||
export interface NotificationEvents {
|
||||
dnsAdd: boolean;
|
||||
dnsUpdate: boolean;
|
||||
@@ -197,6 +200,10 @@ export interface AppSettings {
|
||||
smtp: SmtpSettings;
|
||||
webhook: WebhookSettings;
|
||||
webPush: WebPushSettings;
|
||||
/** Which channels hear which notifications. */
|
||||
routing: RoutingSettings;
|
||||
/** Re-notifying about alerts nobody has acknowledged. */
|
||||
escalation: EscalationSettings;
|
||||
notifications: NotificationEvents;
|
||||
providerColors: ProviderColors;
|
||||
integrationColors: IntegrationColors;
|
||||
@@ -220,6 +227,8 @@ const DEFAULTS: AppSettings = {
|
||||
smtp: { enabled: false, host: "", port: 587, secure: false, username: "", password: "", from: "", to: "" },
|
||||
webhook: { enabled: false, url: "", secret: "" },
|
||||
webPush: { enabled: false, contact: "" },
|
||||
routing: { rules: [] },
|
||||
escalation: { enabled: false, steps: [] },
|
||||
notifications: {
|
||||
dnsAdd: true,
|
||||
dnsUpdate: true,
|
||||
|
||||
@@ -0,0 +1,450 @@
|
||||
/**
|
||||
* Who hears what (routing rules and browsers' wishes) and escalation of alerts nobody has acknowledged.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-routing-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, pushSubscriptions, alertIncidents, auditLog, notificationQueue } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { ruleMatches, firstMatchingRule, pushAccepts, metaOfConditionKey } = await import("../src/services/notifyRouting.js");
|
||||
const { dueEscalations, runEscalation } = await import("../src/services/escalation.js");
|
||||
const { audienceFor, notifyHealthIssues, notifyHealthRecovered, flushQuietHoursQueue, notify } = await import("../src/services/notify.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { diffConditions } = await import("../src/services/healthMonitor.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { setAck } = await import("../src/services/alertAcks.js");
|
||||
const { settingsRouter } = await import("../src/routes/settings.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
type Rule = import("../src/services/notifyRouting.js").RoutingRule;
|
||||
const rule = (extra: Partial<Rule>): Rule => ({ id: "r", name: "rule", enabled: true, categories: [], minSeverity: null, tags: [], channels: [], ...extra });
|
||||
|
||||
describe("routing rules", () => {
|
||||
test("a rule matches on kind, severity and the server's tags; asking about something the notification doesn't say never matches", () => {
|
||||
assert.equal(ruleMatches(rule({}), {}), true, "a rule that asks nothing matches everything");
|
||||
assert.equal(ruleMatches(rule({ categories: ["disk"] }), { category: "disk" }), true);
|
||||
assert.equal(ruleMatches(rule({ categories: ["disk"] }), { category: "offline" }), false);
|
||||
assert.equal(ruleMatches(rule({ categories: ["disk"] }), {}), false, "no kind to compare");
|
||||
assert.equal(ruleMatches(rule({ minSeverity: "critical" }), { severity: "warning" }), false);
|
||||
assert.equal(ruleMatches(rule({ minSeverity: "critical" }), { severity: "critical" }), true);
|
||||
assert.equal(ruleMatches(rule({ minSeverity: "warning" }), { severity: "critical" }), true);
|
||||
assert.equal(ruleMatches(rule({ tags: ["prod"] }), { tags: ["Prod", "db"] }), true, "tags compare without regard to case");
|
||||
assert.equal(ruleMatches(rule({ tags: ["prod"] }), { tags: ["lab"] }), false);
|
||||
assert.equal(ruleMatches(rule({ tags: ["prod"] }), {}), false);
|
||||
assert.equal(ruleMatches(rule({ enabled: false }), {}), false);
|
||||
});
|
||||
|
||||
test("the first rule that matches decides; none matching is no decision", () => {
|
||||
const rules = [rule({ id: "a", categories: ["disk"], channels: ["ntfy"] }), rule({ id: "b", minSeverity: "critical", channels: ["gotify", "webPush"] }), rule({ id: "c", channels: [] })];
|
||||
assert.equal(firstMatchingRule(rules, { category: "disk", severity: "critical" })!.id, "a", "order matters");
|
||||
assert.equal(firstMatchingRule(rules, { category: "offline", severity: "critical" })!.id, "b");
|
||||
assert.equal(firstMatchingRule(rules, { category: "offline", severity: "warning" })!.id, "c");
|
||||
assert.equal(firstMatchingRule([rules[0]], { category: "offline" }), null);
|
||||
});
|
||||
|
||||
test("a problem's key says what kind it is and which server", () => {
|
||||
assert.deepEqual(metaOfConditionKey("offline:server:7", "critical"), { category: "offline", severity: "critical", serverId: 7 });
|
||||
assert.deepEqual(metaOfConditionKey("disk:server:7:/data"), { category: "disk", severity: "warning", serverId: 7 });
|
||||
assert.deepEqual(metaOfConditionKey("failed-units:server:2"), { category: "services", severity: "warning", serverId: 2 });
|
||||
assert.deepEqual(metaOfConditionKey("disk:proxmox:3:local"), { category: "disk", severity: "warning", serverId: null });
|
||||
assert.deepEqual(metaOfConditionKey("automation:semaphore:1:5"), { category: "automation", severity: "warning", serverId: null });
|
||||
});
|
||||
|
||||
test("a browser's wishes: a floor on severity and a list of kinds; what a notification doesn't say can't exclude it", () => {
|
||||
assert.equal(pushAccepts({ minSeverity: null, categories: null }, { category: "disk", severity: "warning" }), true);
|
||||
assert.equal(pushAccepts({ minSeverity: "critical", categories: null }, { severity: "warning" }), false);
|
||||
assert.equal(pushAccepts({ minSeverity: "critical", categories: null }, { severity: "critical" }), true);
|
||||
assert.equal(pushAccepts({ minSeverity: null, categories: JSON.stringify(["offline", "disk"]) }, { category: "backup" }), false);
|
||||
assert.equal(pushAccepts({ minSeverity: null, categories: JSON.stringify(["offline", "disk"]) }, { category: "disk" }), true);
|
||||
assert.equal(pushAccepts({ minSeverity: "critical", categories: JSON.stringify(["disk"]) }, undefined), true, "no meta, no judgement");
|
||||
assert.equal(pushAccepts({ minSeverity: null, categories: JSON.stringify(["disk"]) }, { severity: "warning" }), true, "no kind, no judgement");
|
||||
assert.equal(pushAccepts({ minSeverity: null, categories: "garbage" }, { category: "backup" }), true, "unreadable wishes are no wishes");
|
||||
});
|
||||
});
|
||||
|
||||
describe("what the health monitor remembers", () => {
|
||||
test("a recovery carries how bad the problem was, so it is routed like the alert was", () => {
|
||||
const first = diffConditions({}, [
|
||||
{ key: "offline:server:1", source: "server", message: "a is down", severity: "critical" },
|
||||
{ key: "disk:server:1:/", source: "server", message: "a disk" },
|
||||
]);
|
||||
assert.equal(first.next["offline:server:1"].severity, "critical");
|
||||
assert.equal(first.next["disk:server:1:/"].severity, undefined);
|
||||
const gone = diffConditions(first.next, []);
|
||||
assert.deepEqual(gone.resolved.map((r) => [r.key, r.severity]).sort(), [["disk:server:1:/", undefined], ["offline:server:1", "critical"]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("escalation: what is due", () => {
|
||||
const steps = [
|
||||
{ afterMinutes: 30, minSeverity: "warning" as const, channels: ["webPush" as const], extraEmail: "", evenInQuietHours: false },
|
||||
{ afterMinutes: 120, minSeverity: "critical" as const, channels: ["smtp" as const], extraEmail: "neighbour@example.com", evenInQuietHours: true },
|
||||
];
|
||||
const now = new Date("2026-10-18T12:00:00Z");
|
||||
const ago = (min: number) => new Date(now.getTime() - min * 60_000).toISOString();
|
||||
const alert = (id: string, severity: "critical" | "warning" | "info", extra: Record<string, unknown> = {}) => ({ id, severity, silenced: false, message: id, source: "Server", ...extra });
|
||||
const run = (alerts: ReturnType<typeof alert>[], incidents: Record<string, { openedAt: string; escalatedStep?: number }>, acknowledged: string[] = []) =>
|
||||
dueEscalations({
|
||||
alerts,
|
||||
acknowledged: new Set(acknowledged),
|
||||
incidents: new Map(Object.entries(incidents).map(([k, v], i) => [k, { id: i + 1, openedAt: v.openedAt, escalatedStep: v.escalatedStep ?? 0 }])),
|
||||
steps,
|
||||
now,
|
||||
}).map((d) => [d.alert.id, d.step]);
|
||||
|
||||
test("a step is due once the alert has been up that long; nothing before", () => {
|
||||
assert.deepEqual(run([alert("a", "warning")], { a: { openedAt: ago(29) } }), []);
|
||||
assert.deepEqual(run([alert("a", "warning")], { a: { openedAt: ago(31) } }), [["a", 1]]);
|
||||
});
|
||||
|
||||
test("a step is sent once, and a neglected alert is told about one step at a time", () => {
|
||||
assert.deepEqual(run([alert("a", "critical")], { a: { openedAt: ago(200), escalatedStep: 0 } }), [["a", 1]], "step 1 first, though step 2 is due too");
|
||||
assert.deepEqual(run([alert("a", "critical")], { a: { openedAt: ago(200), escalatedStep: 1 } }), [["a", 2]]);
|
||||
assert.deepEqual(run([alert("a", "critical")], { a: { openedAt: ago(200), escalatedStep: 2 } }), []);
|
||||
});
|
||||
|
||||
test("a step that asks for a worse severity is passed over, and later ones still apply", () => {
|
||||
// a warning is never bad enough for step 2, so it ends after step 1
|
||||
assert.deepEqual(run([alert("w", "warning")], { w: { openedAt: ago(500), escalatedStep: 1 } }), []);
|
||||
const critOnlyFirst = [{ ...steps[0], minSeverity: "critical" as const }, { ...steps[1], minSeverity: "warning" as const }];
|
||||
const due = dueEscalations({ alerts: [alert("w", "warning")], acknowledged: new Set(), incidents: new Map([["w", { id: 1, openedAt: ago(500), escalatedStep: 0 }]]), steps: critOnlyFirst, now });
|
||||
assert.deepEqual(due.map((d) => d.step), [2], "step 1 wasn't for it, step 2 was");
|
||||
});
|
||||
|
||||
test("an acknowledgement stops it; so do maintenance, being held back by another alert, and info", () => {
|
||||
assert.deepEqual(run([alert("a", "critical")], { a: { openedAt: ago(200) } }, ["a"]), []);
|
||||
assert.deepEqual(run([alert("a", "critical", { silenced: true })], { a: { openedAt: ago(200) } }), []);
|
||||
assert.deepEqual(run([alert("a", "info")], { a: { openedAt: ago(200) } }), []);
|
||||
assert.deepEqual(run([alert("a", "critical")], {}), [], "no incident recorded yet, nothing to measure from");
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the channels", () => {
|
||||
type Hit = { path: string; body: string };
|
||||
const hits: Record<"gotify" | "ntfy" | "webhook", Hit[]> = { gotify: [], ntfy: [], webhook: [] };
|
||||
const fake: Server[] = [];
|
||||
const base: Record<string, string> = {};
|
||||
let proddy = 0;
|
||||
|
||||
const text = (h: Hit) => {
|
||||
try {
|
||||
const j = JSON.parse(h.body);
|
||||
return String(j.message ?? j.content ?? h.body);
|
||||
} catch {
|
||||
return h.body;
|
||||
}
|
||||
};
|
||||
const clear = () => {
|
||||
for (const k of Object.keys(hits) as (keyof typeof hits)[]) hits[k].length = 0;
|
||||
};
|
||||
|
||||
before(async () => {
|
||||
for (const name of ["gotify", "ntfy", "webhook"] as const) {
|
||||
const s = createServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
hits[name].push({ path: req.url ?? "", body: Buffer.concat(chunks).toString() });
|
||||
res.statusCode = 200;
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => s.listen(0, "127.0.0.1", r));
|
||||
fake.push(s);
|
||||
base[name] = `http://127.0.0.1:${(s.address() as AddressInfo).port}`;
|
||||
}
|
||||
await updateSettings({
|
||||
gotify: { enabled: true, url: base.gotify, token: "g" },
|
||||
ntfy: { enabled: true, url: base.ntfy, topic: "lab" },
|
||||
webhook: { enabled: true, url: base.webhook },
|
||||
});
|
||||
proddy = (await db.insert(servers).values({ name: "prod-db", apiTokenHash: "p", apiTokenPrefix: "p", tags: JSON.stringify(["prod"]) }).returning({ id: servers.id }))[0].id;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
for (const s of fake) s.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const offline = (id: number) => ({ key: `offline:server:${id}`, message: `server ${id} is offline`, severity: "critical" });
|
||||
const disk = (id: number) => ({ key: `disk:server:${id}:/`, message: `disk on ${id} is full`, severity: "warning" });
|
||||
|
||||
test("with no rules everything goes everywhere, in one message, as it always did", async () => {
|
||||
clear();
|
||||
await notifyHealthIssues([offline(proddy), disk(proddy)]);
|
||||
for (const k of ["gotify", "ntfy", "webhook"] as const) {
|
||||
assert.equal(hits[k].length, 1, k);
|
||||
assert.match(text(hits[k][0]), /2 new issues/);
|
||||
}
|
||||
});
|
||||
|
||||
test("a rule sends one kind to one channel; the rest still goes everywhere, as a message of its own", async () => {
|
||||
await updateSettings({ routing: { rules: [rule({ id: "d", name: "Disks to ntfy", categories: ["disk"], channels: ["ntfy"] })] } });
|
||||
clear();
|
||||
await notifyHealthIssues([offline(proddy), disk(proddy)]);
|
||||
assert.equal(hits.gotify.length, 1);
|
||||
assert.match(text(hits.gotify[0]), /1 new issue:[\s\S]*is offline/);
|
||||
assert.doesNotMatch(text(hits.gotify[0]), /disk/);
|
||||
assert.equal(hits.webhook.length, 1);
|
||||
assert.equal(hits.ntfy.length, 2, "its own message for the disk, and the offline one with the rest");
|
||||
assert.ok(hits.ntfy.some((h) => /disk on/.test(text(h)) && !/offline/.test(text(h))));
|
||||
});
|
||||
|
||||
test("a rule can mute: nobody is told, and nothing is queued", async () => {
|
||||
await updateSettings({ routing: { rules: [rule({ id: "m", name: "Prod disks: nobody", categories: ["disk"], tags: ["prod"], channels: [] })] } });
|
||||
clear();
|
||||
await notifyHealthIssues([disk(proddy)]);
|
||||
assert.deepEqual([hits.gotify.length, hits.ntfy.length, hits.webhook.length], [0, 0, 0]);
|
||||
// the same disk problem on a server without the tag is not muted
|
||||
const plain = (await db.insert(servers).values({ name: "plain", apiTokenHash: "q", apiTokenPrefix: "q" }).returning({ id: servers.id }))[0].id;
|
||||
await notifyHealthIssues([disk(plain)]);
|
||||
assert.equal(hits.gotify.length, 1);
|
||||
});
|
||||
|
||||
test("a recovery follows the route its alert took", async () => {
|
||||
await updateSettings({ routing: { rules: [rule({ id: "c", name: "Critical to webhook", minSeverity: "critical", channels: ["webhook"] })] } });
|
||||
clear();
|
||||
await notifyHealthRecovered([{ key: `offline:server:${proddy}`, message: "back", severity: "critical" }]);
|
||||
assert.deepEqual([hits.gotify.length, hits.ntfy.length, hits.webhook.length], [0, 0, 1]);
|
||||
clear();
|
||||
await notifyHealthRecovered([{ key: `disk:server:${proddy}:/`, message: "ok", severity: "warning" }]);
|
||||
assert.deepEqual([hits.gotify.length, hits.ntfy.length, hits.webhook.length], [1, 1, 1], "a warning isn't covered by that rule, so it goes everywhere");
|
||||
});
|
||||
|
||||
test("a rule that is switched off does nothing", async () => {
|
||||
await updateSettings({ routing: { rules: [rule({ id: "x", enabled: false, channels: [] })] } });
|
||||
clear();
|
||||
await notifyHealthIssues([offline(proddy)]);
|
||||
assert.equal(hits.webhook.length, 1);
|
||||
});
|
||||
|
||||
test("held for quiet hours, a notification is still only for the channels its rule picked", async () => {
|
||||
await updateSettings({ routing: { rules: [rule({ id: "q", categories: ["backup"], channels: ["ntfy"] })] }, quietHours: { enabled: true, start: "00:00", end: "23:59" } });
|
||||
clear();
|
||||
await notify("Backups", "a backup is stale", { category: "backup", severity: "warning" });
|
||||
assert.deepEqual([hits.gotify.length, hits.ntfy.length, hits.webhook.length], [0, 0, 0], "held");
|
||||
const queued = await db.select().from(notificationQueue);
|
||||
assert.equal(queued.length, 1);
|
||||
assert.ok(queued[0].audience, "who it was for is kept");
|
||||
await updateSettings({ quietHours: { enabled: false } });
|
||||
assert.equal(await flushQuietHoursQueue(), 1);
|
||||
assert.deepEqual([hits.gotify.length, hits.ntfy.length, hits.webhook.length], [0, 1, 0]);
|
||||
assert.equal((await db.select().from(notificationQueue)).length, 0);
|
||||
});
|
||||
|
||||
test("browsers' wishes narrow who a push goes to, and only when somebody has stated any", async () => {
|
||||
await updateSettings({ routing: { rules: [] } });
|
||||
const [u] = await db.insert(users).values({ oidcSub: "u1", name: "U", role: "operator" }).returning({ id: users.id });
|
||||
const sub = (n: string, extra: Partial<typeof pushSubscriptions.$inferInsert>) => db.insert(pushSubscriptions).values({ userId: u.id, endpoint: `https://push.example/${n}`, p256dh: "k", auth: "a", ...extra }).returning({ id: pushSubscriptions.id });
|
||||
const [all] = await sub("all", {});
|
||||
assert.equal((await audienceFor({ category: "disk", severity: "warning" })).pushIds, null, "nobody asked for less");
|
||||
const [critical] = await sub("critical", { minSeverity: "critical" });
|
||||
const [disks] = await sub("disks", { categories: JSON.stringify(["disk"]) });
|
||||
const a = await audienceFor({ category: "disk", severity: "warning" });
|
||||
assert.deepEqual([...a.pushIds!].sort(), [all.id, disks.id].sort(), "the one wanting only critical stays out");
|
||||
const b = await audienceFor({ category: "backup", severity: "critical" });
|
||||
assert.deepEqual([...b.pushIds!].sort(), [all.id, critical.id].sort(), "and the one wanting only disks stays out of backups");
|
||||
assert.equal((await audienceFor()).pushIds, null, "a notification that says nothing about itself goes to all");
|
||||
});
|
||||
});
|
||||
|
||||
describe("escalation, end to end", () => {
|
||||
const hits: { path: string; body: string }[] = [];
|
||||
let fakeHook: Server;
|
||||
|
||||
before(async () => {
|
||||
fakeHook = createServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
hits.push({ path: req.url ?? "", body: Buffer.concat(chunks).toString() });
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => fakeHook.listen(0, "127.0.0.1", r));
|
||||
await db.delete(servers);
|
||||
await db.insert(servers).values({ name: "late", apiTokenHash: "l", apiTokenPrefix: "l", lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString() });
|
||||
await updateSettings({
|
||||
gotify: { enabled: false },
|
||||
ntfy: { enabled: false },
|
||||
webhook: { enabled: true, url: `http://127.0.0.1:${(fakeHook.address() as AddressInfo).port}/hook` },
|
||||
routing: { rules: [] },
|
||||
quietHours: { enabled: false },
|
||||
escalation: {
|
||||
enabled: true,
|
||||
steps: [
|
||||
{ afterMinutes: 30, minSeverity: "warning", channels: ["webhook"], extraEmail: "", evenInQuietHours: false },
|
||||
{ afterMinutes: 90, minSeverity: "critical", channels: ["webhook"], extraEmail: "", evenInQuietHours: true },
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
after(() => fakeHook.close());
|
||||
|
||||
const setAge = async (minutes: number) => {
|
||||
await db.update(alertIncidents).set({ openedAt: new Date(Date.now() - minutes * 60_000).toISOString() });
|
||||
};
|
||||
const text = () => hits.map((h) => JSON.parse(h.body).content as string);
|
||||
|
||||
test("nothing happens until the alert has been up long enough; then the steps go out one by one, each once", async () => {
|
||||
invalidateAlertsCache();
|
||||
await getAlerts(true);
|
||||
await new Promise((r) => setTimeout(r, 300)); // the incident is recorded in the background
|
||||
assert.equal(await runEscalation(), 0, "just raised");
|
||||
assert.equal(hits.length, 0);
|
||||
|
||||
await setAge(40);
|
||||
assert.equal(await runEscalation(), 1);
|
||||
assert.equal(hits.length, 1);
|
||||
assert.match(text()[0], /Still not acknowledged/);
|
||||
assert.match(text()[0], /late hasn't reported/);
|
||||
assert.match(text()[0], /after 30 minutes/);
|
||||
assert.equal(await runEscalation(), 0, "step 1 is not repeated");
|
||||
|
||||
await setAge(100);
|
||||
assert.equal(await runEscalation(), 1);
|
||||
assert.match(text()[1], /after 90 minutes/);
|
||||
assert.equal(await runEscalation(), 0, "and that was the last step");
|
||||
const [inc] = await db.select().from(alertIncidents);
|
||||
assert.equal(inc.escalatedStep, 2);
|
||||
assert.equal((await db.select().from(auditLog)).filter((e) => e.action === "escalate").length, 2);
|
||||
});
|
||||
|
||||
test("an acknowledgement stops it, and so does switching it off", async () => {
|
||||
await db.update(alertIncidents).set({ escalatedStep: 0, escalatedAt: null });
|
||||
await setAge(500);
|
||||
const before = hits.length;
|
||||
const alert = (await getAlerts(false)).alerts.find((a) => a.category === "offline")!;
|
||||
await setAck({ alertId: alert.id, note: "on it", assigneeId: null, hours: null, actor: "Ada" });
|
||||
assert.equal(await runEscalation(), 0);
|
||||
await db.delete((await import("../src/db/schema.js")).alertAcks);
|
||||
await updateSettings({ escalation: { enabled: false } });
|
||||
assert.equal(await runEscalation(), 0);
|
||||
assert.equal(hits.length, before);
|
||||
await updateSettings({ escalation: { enabled: true } });
|
||||
assert.equal(await runEscalation(), 1, "and it carries on once it is on and nobody is on it");
|
||||
});
|
||||
});
|
||||
|
||||
describe("saving the settings", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values({ oidcSub: "admin", name: "Ada", role: "admin" }).onConflictDoNothing();
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/settings", settingsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/settings`;
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
const put = (body: unknown) => fetch(base, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
|
||||
const step = (afterMinutes: number, extra: Record<string, unknown> = {}) => ({ afterMinutes, minSeverity: "critical", channels: ["webPush"], extraEmail: "", evenInQuietHours: false, ...extra });
|
||||
|
||||
test("rules are kept, with their tags written the way server tags are", async () => {
|
||||
const res = await put({ routing: { rules: [{ id: "a1", name: "Prod", enabled: true, categories: ["disk"], minSeverity: "critical", tags: [" Prod ", "Db Servers"], channels: ["ntfy"] }] } });
|
||||
assert.equal(res.status, 200);
|
||||
const saved = (await (await fetch(base)).json()).settings.routing.rules[0];
|
||||
assert.deepEqual(saved.tags, ["db-servers", "prod"]);
|
||||
});
|
||||
|
||||
test("a rule that isn't one is refused", async () => {
|
||||
assert.equal((await put({ routing: { rules: [{ id: "a", name: "x", enabled: true, categories: ["toast"], minSeverity: null, tags: [], channels: [] }] } })).status, 400);
|
||||
assert.equal((await put({ routing: { rules: [{ id: "a", name: "x", enabled: true, categories: [], minSeverity: null, tags: [], channels: ["carrier-pigeon"] }] } })).status, 400);
|
||||
assert.equal((await put({ routing: { rules: [{ id: "a", name: "x", enabled: true, categories: [], minSeverity: null, tags: ["not a tag!"], channels: [] }] } })).status, 400);
|
||||
});
|
||||
|
||||
test("escalation steps must be in time order, say who to tell, and be few", async () => {
|
||||
assert.equal((await put({ escalation: { enabled: true, steps: [step(30), step(120, { extraEmail: "x@example.com", channels: [] })] } })).status, 200);
|
||||
assert.equal((await put({ escalation: { enabled: true, steps: [step(60), step(30)] } })).status, 400, "out of order");
|
||||
assert.equal((await put({ escalation: { enabled: true, steps: [step(30, { channels: [], extraEmail: "" })] } })).status, 400, "nobody to tell");
|
||||
assert.equal((await put({ escalation: { enabled: true, steps: [step(30, { extraEmail: "not an address" })] } })).status, 400);
|
||||
assert.equal((await put({ escalation: { enabled: true, steps: [step(10), step(20), step(30), step(40)] } })).status, 400, "at most three");
|
||||
assert.equal((await put({ escalation: { enabled: true, steps: [step(2)] } })).status, 400, "not sooner than five minutes");
|
||||
});
|
||||
});
|
||||
|
||||
describe("ntfy titles", () => {
|
||||
test("a title with an em dash can be sent as a header; plain ones are left as they are", async () => {
|
||||
const { headerSafe } = await import("../src/services/notify.js");
|
||||
assert.equal(headerSafe("Plain title"), "Plain title");
|
||||
const encoded = headerSafe("Homelab Manager — Health Alert");
|
||||
assert.match(encoded, /^=\?UTF-8\?B\?[A-Za-z0-9+/=]+\?=$/);
|
||||
assert.equal(Buffer.from(encoded.slice(10, -2), "base64").toString("utf8"), "Homelab Manager — Health Alert");
|
||||
assert.doesNotThrow(() => new Headers({ Title: encoded }), "and the runtime takes it");
|
||||
assert.throws(() => new Headers({ Title: "Homelab Manager — Health Alert" }), "which it would not take as it was");
|
||||
});
|
||||
});
|
||||
|
||||
describe("a browser's preferences, over the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "p1";
|
||||
let mine = 0;
|
||||
|
||||
before(async () => {
|
||||
const { pushRouter } = await import("../src/routes/push.js");
|
||||
const made = await db.insert(users).values([{ oidcSub: "p1", name: "P1", role: "viewer" }, { oidcSub: "p2", name: "P2", role: "viewer" }]).returning({ id: users.id });
|
||||
mine = (await db.insert(pushSubscriptions).values({ userId: made[0].id, endpoint: "https://push.example/mine", p256dh: "k", auth: "a" }).returning({ id: pushSubscriptions.id }))[0].id;
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/push", pushRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/push`;
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
const patch = (id: number, body: unknown) => fetch(`${base}/${id}`, { method: "PATCH", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
|
||||
|
||||
test("a person sets what reaches a browser and reads it back; someone else's browser is not theirs to set", async () => {
|
||||
const first = await patch(mine, { minSeverity: "critical", categories: ["offline", "disk"] });
|
||||
assert.equal(first.status, 200, JSON.stringify(await first.clone().json()));
|
||||
const list = (await (await fetch(`${base}/mine`)).json()).subscriptions[0];
|
||||
assert.equal(list.minSeverity, "critical");
|
||||
assert.deepEqual(list.categories, ["offline", "disk"]);
|
||||
assert.equal((await patch(mine, { minSeverity: null, categories: [] })).status, 200, "an empty list is every kind");
|
||||
const cleared = (await (await fetch(`${base}/mine`)).json()).subscriptions[0];
|
||||
assert.deepEqual([cleared.minSeverity, cleared.categories], [null, null]);
|
||||
who = "p2";
|
||||
assert.equal((await patch(mine, { minSeverity: "critical", categories: null })).status, 404);
|
||||
who = "p1";
|
||||
assert.equal((await patch(mine, { minSeverity: "info", categories: null })).status, 400, "info never notifies, so it isn't a floor");
|
||||
assert.equal((await patch(mine, { minSeverity: null, categories: ["toast"] })).status, 400);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user