Notification routing rules, per-browser push wishes, escalation of unacknowledged alerts

- Routing: rules pick channels by kind, severity and server tags (first match wins, none = nobody); mixed batches split per route; recoveries follow their alert; quiet-hours digest keeps each notification's audience
- Push: each subscribed browser can ask for a minimum severity and some kinds
- Escalation: up to three timed steps re-notify alerts nobody has acknowledged, through chosen channels and an optional extra email; an acknowledgement stops them
- Fix ntfy: titles with an em dash made every ntfy send fail (header not Latin-1); now RFC 2047 encoded

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 19:44:45 +02:00
1 parent 9caa414829
commit 43fc5e0272
21 files changed
+4769 -55

No files matched your search

+8
View File
@@ -64,6 +64,8 @@ export const notificationQueue = sqliteTable("notification_queue", {
id: integer("id").primaryKey({ autoIncrement: true }),
title: text("title").notNull(),
message: text("message").notNull(),
/** JSON: who it is for (the channels and browsers a routing rule picked); null = everyone, as before routing existed. */
audience: text("audience"),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
@@ -119,6 +121,10 @@ export const pushSubscriptions = sqliteTable("push_subscriptions", {
.notNull()
.default(sql`(current_timestamp)`),
lastSuccessAt: text("last_success_at"),
/** This browser only wants alerts at least this bad ("warning" or "critical"); null = every severity. */
minSeverity: text("min_severity"),
/** JSON string[] of the kinds of alert it wants; null = every kind. */
categories: text("categories"),
});
// "A needs B", for things that can be down: a server or an integration each. Polymorphic like maintenance_windows (no foreign key); an edge whose
@@ -678,6 +684,8 @@ export const alertIncidents = sqliteTable(
resolvedAt: text("resolved_at"), // ISO; null while it is open
lastSeenAt: text("last_seen_at").notNull(), // ISO
missingSince: text("missing_since"), // ISO; set while it has been absent from the last look(s), before it is called over
escalatedStep: integer("escalated_step").notNull().default(0), // the highest escalation step already sent for this incident (0 = none)
escalatedAt: text("escalated_at"), // ISO; when that was
},
(t) => [index("alert_incidents_key_idx").on(t.alertKey), index("alert_incidents_opened_idx").on(t.openedAt), index("alert_incidents_target_idx").on(t.targetType, t.targetId)],
);
+2
View File
@@ -19,6 +19,7 @@ import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceScheduler.js";
import { initMaintenanceScheduleScheduler } from "./services/maintenanceScheduleScheduler.js";
import { initUpdateRuns } from "./services/updateRuns.js";
import { initEscalation } from "./services/escalation.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
import { initIncidentRecorder } from "./services/incidents.js";
import { getAlerts } from "./services/alerts.js";
@@ -57,6 +58,7 @@ await initMaintenanceScheduleScheduler();
initUpdateRuns();
await initSelfBackupScheduler();
initIncidentRecorder(() => getAlerts(false));
initEscalation();
createApp().listen(env.port, () => {
console.log(`homelab-manager listening on port ${env.port}`);
+21 -1
View File
@@ -7,6 +7,7 @@ import { requireAuth } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { getSettings } from "../services/settingsStore.js";
import { checkEndpoint, getVapidKeys, sendPush } from "../services/webPush.js";
import { CATEGORIES, parseCategories } from "../services/notifyRouting.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const pushRouter = Router();
@@ -45,7 +46,7 @@ pushRouter.get("/mine", asyncHandler(async (req, res) => {
const { webPush } = await getSettings();
res.json({
enabled: webPush.enabled,
subscriptions: rows.map((r) => ({ id: r.id, label: r.label, host: hostOf(r.endpoint), endpoint: r.endpoint, createdAt: r.createdAt, lastSuccessAt: r.lastSuccessAt })),
subscriptions: rows.map((r) => ({ id: r.id, label: r.label, host: hostOf(r.endpoint), endpoint: r.endpoint, createdAt: r.createdAt, lastSuccessAt: r.lastSuccessAt, minSeverity: r.minSeverity === "critical" || r.minSeverity === "warning" ? r.minSeverity : null, categories: parseCategories(r.categories) })),
});
}));
@@ -74,6 +75,25 @@ pushRouter.post("/subscribe", asyncHandler(async (req, res) => {
res.status(201).json({ id: row.id });
}));
// What a browser wants to be told: only the worse alerts, only some kinds. Its owner's to say (an administrator may too).
const prefsSchema = z.object({
minSeverity: z.enum(["warning", "critical"]).nullable(),
categories: z.array(z.enum(CATEGORIES)).max(10).nullable(),
});
pushRouter.patch("/:id", asyncHandler(async (req, res) => {
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
const parsed = prefsSchema.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Choose a severity (or any) and the kinds of alert (or all).", details: parsed.success ? undefined : parsed.error.flatten() });
const me = req.currentUser!;
const [row] = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.id, id)).limit(1);
if (!row || (row.userId !== me.id && me.role !== "admin")) return res.status(404).json({ error: "not_found" });
const categories = parsed.data.categories && parsed.data.categories.length > 0 ? parsed.data.categories : null;
await db.update(pushSubscriptions).set({ minSeverity: parsed.data.minSeverity, categories: categories ? JSON.stringify(categories) : null }).where(eq(pushSubscriptions.id, id));
await recordAudit({ actor: me, category: "push", action: "preferences", targetType: "push_subscription", targetId: id, detail: { host: hostOf(row.endpoint), label: row.label, minSeverity: parsed.data.minSeverity, categories } });
res.json({ minSeverity: parsed.data.minSeverity, categories });
}));
// Stops a browser being written to. A person removes their own; an administrator may remove anyone's.
pushRouter.delete("/:id", asyncHandler(async (req, res) => {
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
+54
View File
@@ -15,6 +15,8 @@ import { scheduleQuietHoursFlush } from "../services/quietHoursScheduler.js";
import { scheduleWeeklyDigest } from "../services/weeklyDigestScheduler.js";
import { buildDigest, sendDigest } from "../services/weeklyDigest.js";
import { enabledChannelCount } from "../services/notify.js";
import { CATEGORIES, CHANNEL_IDS } from "../services/notifyRouting.js";
import { InvalidTagError, normalizeTags } from "../services/serverTags.js";
import { testGotify, testNtfy, testSmtp, testWebhook, flushQuietHoursQueue } from "../services/notify.js";
import { listQueuedNotifications } from "../services/notificationQueue.js";
import { buildExportPayload, encryptExport, decryptExport, applyImportPayload, type EncryptedExportFile } from "../services/configBackup.js";
@@ -47,7 +49,51 @@ settingsRouter.get("/display", asyncHandler(async (_req, res) => {
res.json({ display });
}));
const channelEnum = z.enum(CHANNEL_IDS);
// Routing rules: which channels hear which notifications. A rule with no channels means "tell nobody" (the Alerts page still shows it).
const routingSchema = z.object({
rules: z
.array(
z.object({
id: z.string().trim().min(1).max(40),
name: z.string().trim().min(1).max(60),
enabled: z.boolean(),
categories: z.array(z.enum(CATEGORIES)).max(10),
minSeverity: z.enum(["warning", "critical"]).nullable(),
tags: z.array(z.string().trim().min(1).max(30)).max(10),
channels: z.array(channelEnum).max(5),
}),
)
.max(30),
});
// Escalation: a few steps in time order. Each must say who to tell — a channel, or an address.
const escalationSchema = z
.object({
enabled: z.boolean(),
steps: z
.array(
z.object({
afterMinutes: z.number().int().min(5).max(10080),
minSeverity: z.enum(["warning", "critical"]),
channels: z.array(channelEnum).max(5),
extraEmail: z.string().trim().max(200).refine((v) => v === "" || /^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(v), "That isn't an email address"),
evenInQuietHours: z.boolean(),
}),
)
.max(3),
})
.superRefine((v, ctx) => {
v.steps.forEach((s, i) => {
if (s.channels.length === 0 && s.extraEmail === "") ctx.addIssue({ code: "custom", path: ["steps", i], message: "Choose a channel to tell, or an address to email." });
if (i > 0 && s.afterMinutes <= v.steps[i - 1].afterMinutes) ctx.addIssue({ code: "custom", path: ["steps", i, "afterMinutes"], message: "Each step must come later than the one before." });
});
});
const updateSchema = z.object({
routing: routingSchema.optional(),
escalation: escalationSchema.optional(),
gotify: z.object({ enabled: z.boolean(), url: z.string(), token: z.string(), priority: z.number() }).partial().optional(),
ntfy: z.object({ enabled: z.boolean(), url: z.string(), topic: z.string(), token: z.string(), priority: z.number() }).partial().optional(),
smtp: z
@@ -126,6 +172,14 @@ settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
return res.status(400).json({ error: "invalid_body", details: parsed.error.flatten() });
}
// Tags in routing rules are written the way server tags are, so they match.
try {
if (parsed.data.routing) for (const rule of parsed.data.routing.rules) rule.tags = normalizeTags(rule.tags);
} catch (err) {
if (err instanceof InvalidTagError) return res.status(400).json({ error: "invalid_tag", message: err.message });
throw err;
}
const before = await getSettings();
const updated = await updateSettings(parsed.data);
+112
View File
@@ -0,0 +1,112 @@
/**
* Escalation: an alert that has been raised for a while and that nobody has acknowledged is told again — through louder channels, and
* perhaps to someone else. Steps are in order ("after 30 minutes: push; after 2 hours: email the neighbour who holds the key"), each is sent
* at most once per incident (one occurrence of an alert; it starts over if the alert goes away and comes back), and an acknowledgement stops
* the lot. Alerts that are silenced for maintenance, or held back because something they depend on is down, never escalate.
*/
import { and, eq, inArray, isNull } from "drizzle-orm";
import { db } from "../db/client.js";
import { alertIncidents } from "../db/schema.js";
import type { Alert } from "./alertTypes.js";
import { getAlerts } from "./alerts.js";
import { loadAcks } from "./alertAcks.js";
import { recordAudit } from "./audit.js";
import { notifyAudience } from "./notify.js";
import { getSettings } from "./settingsStore.js";
import { severityRank, type EscalationStep } from "./notifyRouting.js";
export interface DueEscalation {
alert: Pick<Alert, "id" | "severity" | "message" | "source">;
incidentId: number;
/** 1-based, in the order the steps are kept. */
step: number;
definition: EscalationStep;
}
export interface OpenIncident {
id: number;
openedAt: string;
escalatedStep: number;
}
/**
* Which escalations are due now. Pure. An alert gets at most one step per look — the first one that is due, applies to its severity and hasn't
* been sent — so a long-neglected alert is told about once per look rather than all at once.
*/
export function dueEscalations(input: {
alerts: Pick<Alert, "id" | "severity" | "silenced" | "message" | "source">[];
acknowledged: Set<string>;
incidents: Map<string, OpenIncident>;
steps: EscalationStep[];
now: Date;
}): DueEscalation[] {
const out: DueEscalation[] = [];
for (const alert of input.alerts) {
if (alert.silenced || alert.severity === "info" || input.acknowledged.has(alert.id)) continue;
const incident = input.incidents.get(alert.id);
if (!incident) continue;
const ageMinutes = (input.now.getTime() - Date.parse(incident.openedAt)) / 60_000;
for (let i = incident.escalatedStep; i < input.steps.length; i++) {
const def = input.steps[i];
if (ageMinutes < def.afterMinutes) break; // steps are in time order: none after this one is due either
if (severityRank(alert.severity) < severityRank(def.minSeverity)) continue; // not bad enough for this step; a later one may be for it
out.push({ alert, incidentId: incident.id, step: i + 1, definition: def });
break;
}
}
return out;
}
const words = (minutes: number) => (minutes >= 120 && minutes % 60 === 0 ? `${minutes / 60} hours` : minutes === 60 ? "an hour" : `${minutes} minutes`);
/** One look: sends whatever is due. Returns how many alerts were escalated. */
export async function runEscalation(now: Date = new Date()): Promise<number> {
const { escalation } = await getSettings();
if (!escalation.enabled || escalation.steps.length === 0) return 0;
const report = await getAlerts(false);
const ids = report.alerts.map((a) => a.id);
if (ids.length === 0) return 0;
const open = await db.select().from(alertIncidents).where(and(isNull(alertIncidents.resolvedAt), inArray(alertIncidents.alertKey, ids)));
const acks = await loadAcks(now);
const due = dueEscalations({
alerts: report.alerts,
acknowledged: new Set([...acks].filter(([, a]) => a.active).map(([id]) => id)),
incidents: new Map(open.map((i) => [i.alertKey, { id: i.id, openedAt: i.openedAt, escalatedStep: i.escalatedStep }])),
steps: escalation.steps,
now,
});
// One message for each step, listing the alerts it is for.
const byStep = new Map<number, DueEscalation[]>();
for (const d of due) byStep.set(d.step, [...(byStep.get(d.step) ?? []), d]);
for (const [step, items] of byStep) {
const def = items[0].definition;
const lines = items.map((d) => `${d.alert.severity === "critical" ? "✕" : "⚠"} ${d.alert.message}`);
const message = `Nobody has acknowledged ${items.length === 1 ? "this alert" : `these ${items.length} alerts`} after ${words(def.afterMinutes)}:\n\n${lines.join("\n")}\n\nAcknowledge ${items.length === 1 ? "it" : "them"} on the Alerts page to stop the reminders.`;
try {
await notifyAudience("Homelab Manager — Still not acknowledged", message, { channels: def.channels, pushIds: null, ...(def.extraEmail ? { email: def.extraEmail } : {}) }, { quietHours: !def.evenInQuietHours });
} catch (err) {
console.error("[escalation] couldn't send step", step, err);
continue; // not marked as sent: the next look tries again
}
for (const d of items) {
await db.update(alertIncidents).set({ escalatedStep: step, escalatedAt: now.toISOString() }).where(eq(alertIncidents.id, d.incidentId));
await recordAudit({ category: "alert", action: "escalate", detail: { alertId: d.alert.id, message: d.alert.message, step, afterMinutes: def.afterMinutes, channels: def.channels, email: def.extraEmail || null } });
}
}
return due.length;
}
let timer: ReturnType<typeof setInterval> | null = null;
/** Looks every five minutes (a little after the incident recorder, which uses the same fresh result). */
export function initEscalation(): void {
if (timer) return;
const tick = () =>
void runEscalation().catch((err) => console.error("[escalation] the look failed:", err));
setTimeout(tick, 120_000).unref();
timer = setInterval(tick, 5 * 60_000);
timer.unref();
}
+5 -5
View File
@@ -224,7 +224,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
// ─── State diff (pure) ──────────────────────────────────────────────────────
/** `covered`: the problem is explained by another that is being reported (a guest down because its host is) — recorded, but not announced on its own. */
export type ActiveState = Record<string, { message: string; source: string; covered?: boolean }>;
export type ActiveState = Record<string, { message: string; source: string; covered?: boolean; severity?: "critical" | "warning" }>;
/** `held` is a set of hierarchical sources that couldn't be read this run; matches the source itself or anything beneath it. */
function isHeld(source: string, held: Set<string>): boolean {
@@ -252,7 +252,7 @@ export function diffConditions(
covered: Set<string> = new Set(),
): {
added: HealthCondition[];
resolved: { key: string; message: string }[];
resolved: { key: string; message: string; severity?: "critical" | "warning" }[];
next: ActiveState;
/** Every problem that began, announced or not. */
newlyDown: HealthCondition[];
@@ -268,12 +268,12 @@ export function diffConditions(
const visible = current.filter((c) => !isSilenced(c.key));
const next: ActiveState = {};
for (const c of visible) next[c.key] = covered.has(c.key) ? { message: c.message, source: c.source, covered: true } : { message: c.message, source: c.source };
for (const c of visible) next[c.key] = { message: c.message, source: c.source, ...(covered.has(c.key) ? { covered: true } : {}), ...(c.severity ? { severity: c.severity } : {}) };
const newlyDown = visible.filter((c) => !(c.key in previous));
// Announced: new and standing alone, or covered until now and no longer.
const added = visible.filter((c) => !covered.has(c.key) && (!(c.key in previous) || previous[c.key].covered));
const resolved: { key: string; message: string }[] = [];
const resolved: { key: string; message: string; severity?: "critical" | "warning" }[] = [];
const ended: { key: string; message: string }[] = [];
for (const [key, entry] of Object.entries(previous)) {
if (key in next) continue;
@@ -282,7 +282,7 @@ export function diffConditions(
next[key] = entry;
} else {
ended.push({ key, message: entry.message });
if (!entry.covered) resolved.push({ key, message: entry.message });
if (!entry.covered) resolved.push({ key, message: entry.message, ...(entry.severity ? { severity: entry.severity } : {}) });
}
}
return { added, resolved, next, newlyDown, ended };
+4 -2
View File
@@ -5,11 +5,13 @@ export interface QueuedNotification {
id: number;
title: string;
message: string;
/** JSON of who it was for; null means everyone. */
audience: string | null;
createdAt: string;
}
export async function enqueueNotification(title: string, message: string): Promise<void> {
await db.insert(notificationQueue).values({ title, message });
export async function enqueueNotification(title: string, message: string, audience: string | null = null): Promise<void> {
await db.insert(notificationQueue).values({ title, message, audience });
}
export async function listQueuedNotifications(): Promise<QueuedNotification[]> {
+170 -38
View File
@@ -3,6 +3,11 @@ import { getSettings, type NotificationEvents } from "./settingsStore.js";
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
import { emitEvent } from "./eventWebhooks.js";
import { sendPush, subscriptionCount } from "./webPush.js";
import { db } from "../db/client.js";
import { inArray } from "drizzle-orm";
import { pushSubscriptions, servers } from "../db/schema.js";
import { parseTags } from "./serverTags.js";
import { firstMatchingRule, metaOfConditionKey, pushAccepts, type ChannelId, type NotifyMeta } from "./notifyRouting.js";
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
@@ -22,13 +27,22 @@ async function sendGotify(title: string, message: string) {
}
}
/**
* A header can only carry Latin-1, and every title here has an em dash. ntfy reads RFC 2047 encoded words in its headers ("=?UTF-8?B?...?="),
* so anything outside plain ASCII is sent that way.
*/
export function headerSafe(text: string): string {
// eslint-disable-next-line no-control-regex
return /^[ -~]*$/.test(text) ? text : `=?UTF-8?B?${Buffer.from(text, "utf8").toString("base64")}?=`;
}
async function sendNtfy(title: string, message: string) {
const { ntfy } = await getSettings();
if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return;
const base = ntfy.url.replace(/\/$/, "");
const headers: Record<string, string> = {
"Content-Type": "text/plain",
Title: title,
Title: headerSafe(title),
Priority: String(ntfy.priority ?? 3),
};
if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`;
@@ -73,18 +87,116 @@ async function sendWebhook(title: string, message: string) {
}
}
async function sendWebPush(title: string, message: string) {
async function sendWebPush(title: string, message: string, only: number[] | null) {
const { webPush } = await getSettings();
if (!webPush.enabled) return;
try {
await sendPush({ title, body: message });
if (only === null) await sendPush({ title, body: message });
else if (only.length > 0) await sendPush({ title, body: message }, await db.select().from(pushSubscriptions).where(inArray(pushSubscriptions.id, only)));
} catch (err) {
console.error("[notify] Web push failed:", err instanceof Error ? err.message : err);
}
}
async function sendToChannels(title: string, message: string): Promise<void> {
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message), sendWebPush(title, message)]);
/** Email to one address, through the email channel's server, whatever the channel's own recipient is (an escalation to someone else). */
async function sendExtraEmail(title: string, message: string, to: string) {
const { smtp } = await getSettings();
if (!smtp.host || !smtp.from) {
console.error("[notify] An escalation wants to email someone, but email isn't set up.");
return;
}
try {
const transporter = nodemailer.createTransport({
host: smtp.host,
port: Number(smtp.port) || 587,
secure: !!smtp.secure,
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
});
await transporter.sendMail({ from: smtp.from, to, subject: title, text: message });
} catch (err) {
console.error("[notify] Extra email failed:", err instanceof Error ? err.message : err);
}
}
// ─── Who a notification is for ─────────────────────────────────────────────
/** The channels and browsers a notification goes to. `null` means every one that is on (the way it was before routing existed). */
export interface Audience {
channels: ChannelId[] | null;
/** The subscribed browsers that want it; null = all of them. */
pushIds: number[] | null;
/** Also email this address. */
email?: string;
}
export const EVERYONE: Audience = { channels: null, pushIds: null };
export const audienceKey = (a: Audience) => JSON.stringify({ c: a.channels ? [...a.channels].sort() : null, p: a.pushIds ? [...a.pushIds].sort((x, y) => x - y) : null, e: a.email ?? null });
/** Told to nobody: a rule said so (the Alerts page still shows it). */
const isMuted = (a: Audience) => a.channels !== null && a.channels.length === 0 && !a.email;
/** Works out who a notification is for: the first routing rule it matches picks the channels, and browsers may turn down what they don't want. */
export async function audienceFor(meta?: NotifyMeta): Promise<Audience> {
if (!meta) return EVERYONE;
const { routing } = await getSettings();
let channels: ChannelId[] | null = null;
if (routing.rules.some((r) => r.enabled)) {
let tags = meta.tags;
if (!tags && meta.serverId != null) {
const [row] = await db.select({ tags: servers.tags }).from(servers).where(inArray(servers.id, [meta.serverId]));
tags = parseTags(row?.tags);
}
channels = firstMatchingRule(routing.rules, { ...meta, tags: tags ?? [] })?.channels ?? null;
}
let pushIds: number[] | null = null;
if (channels === null || channels.includes("webPush")) {
const subs = await db.select({ id: pushSubscriptions.id, minSeverity: pushSubscriptions.minSeverity, categories: pushSubscriptions.categories }).from(pushSubscriptions);
// Only when somebody has said what they want; otherwise everyone gets it, as ever.
if (subs.some((s) => s.minSeverity || s.categories)) pushIds = subs.filter((s) => pushAccepts(s, meta)).map((s) => s.id);
}
return { channels, pushIds };
}
async function sendToChannels(title: string, message: string, audience: Audience = EVERYONE): Promise<void> {
const to = (c: ChannelId) => audience.channels === null || audience.channels.includes(c);
await Promise.all([
to("gotify") ? sendGotify(title, message) : undefined,
to("ntfy") ? sendNtfy(title, message) : undefined,
to("smtp") ? sendSmtp(title, message) : undefined,
to("webhook") ? sendWebhook(title, message) : undefined,
to("webPush") ? sendWebPush(title, message, audience.pushIds) : undefined,
audience.email ? sendExtraEmail(title, message, audience.email) : undefined,
]);
}
/**
* Sends to an audience that is already worked out: held for the quiet-hours digest if it is quiet (unless told not to), nowhere if a rule
* said nobody. Escalations use this with their own channels.
*/
export async function notifyAudience(title: string, message: string, audience: Audience, options: { quietHours?: boolean } = {}): Promise<void> {
if (isMuted(audience)) return;
if (options.quietHours !== false && (await isQuietHoursActive())) {
await enqueueNotification(title, message, audience.channels === null && audience.pushIds === null && !audience.email ? null : JSON.stringify(audience));
return;
}
await sendToChannels(title, message, audience);
}
/** Items that may each be for a different audience (a rule can send a disk warning one place and a server outage another): sent as one message per audience. */
async function notifyGrouped<T>(items: T[], metaOf: (item: T) => NotifyMeta, build: (group: T[]) => { title: string; message: string }): Promise<void> {
const groups = new Map<string, { audience: Audience; items: T[] }>();
for (const item of items) {
const audience = await audienceFor(metaOf(item));
const key = audienceKey(audience);
const group = groups.get(key) ?? { audience, items: [] };
group.items.push(item);
groups.set(key, group);
}
for (const group of groups.values()) {
const { title, message } = build(group.items);
await notifyAudience(title, message, group.audience);
}
}
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
@@ -133,23 +245,32 @@ export async function enabledChannelCount(): Promise<number> {
].filter(Boolean).length;
}
export async function notify(title: string, message: string): Promise<void> {
if (await isQuietHoursActive()) {
await enqueueNotification(title, message);
return;
}
await sendToChannels(title, message);
/** `meta` says what it is about, so routing rules and browsers' wishes can apply; without it, it goes to everyone. */
export async function notify(title: string, message: string, meta?: NotifyMeta): Promise<void> {
await notifyAudience(title, message, await audienceFor(meta));
}
/** Called by the daily scheduler at quiet-hours end (or manually) — sends everything queued during the window as one digest, then clears it. */
export async function flushQuietHoursQueue(): Promise<number> {
const queued = await listQueuedNotifications();
if (queued.length === 0) return 0;
const sections = queued.map((q) => `— ${q.title} —\n${q.message}`);
await sendToChannels(
"Homelab Manager — Notifications from quiet hours",
`${queued.length} notification${queued.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
);
// One digest for each audience the held notifications were for.
const groups = new Map<string, typeof queued>();
for (const q of queued) groups.set(q.audience ?? "", [...(groups.get(q.audience ?? "") ?? []), q]);
for (const [stored, items] of groups) {
let audience: Audience = EVERYONE;
try {
if (stored) audience = JSON.parse(stored) as Audience;
} catch {
// unreadable: everyone, as before
}
const sections = items.map((q) => `— ${q.title} —\n${q.message}`);
await sendToChannels(
"Homelab Manager — Notifications from quiet hours",
`${items.length} notification${items.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
audience,
);
}
await clearQueuedNotifications();
return queued.length;
}
@@ -172,7 +293,7 @@ export async function testGotify(cfg: { url: string; token: string; priority?: n
export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise<void> {
const base = cfg.url.replace(/\/$/, "");
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: "Homelab Manager — Test", Priority: String(cfg.priority ?? 3) };
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: headerSafe("Homelab Manager — Test"), Priority: String(cfg.priority ?? 3) };
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, {
method: "POST",
@@ -263,7 +384,7 @@ export async function notifySecretExpiry(
`Couldn't read the live certificate for ${checkFailures.length} monitored secret${checkFailures.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
);
}
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"));
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"), { category: "expiry", severity: expiring.some((s) => s.status === "expired") ? "critical" : "warning" });
}
export async function notifyDockerUpdates(
@@ -277,6 +398,7 @@ export async function notifyDockerUpdates(
await notify(
"Homelab Manager — Docker Updates Available",
`${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`,
{ category: "updates", severity: "warning" },
);
}
@@ -291,6 +413,7 @@ export async function notifyProxmoxBackupFailure(
await notify(
"Homelab Manager — Proxmox Backup Failed",
`${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
{ category: "backup", severity: "warning" },
);
}
@@ -303,6 +426,7 @@ export async function notifyProxmoxUncoveredGuests(
await notify(
"Homelab Manager — Proxmox Guests Without Backup Coverage",
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
{ category: "backup", severity: "warning" },
);
}
@@ -312,6 +436,7 @@ export async function notifyStaleBackups(stale: { message: string }[]): Promise<
await notify(
"Homelab Manager — Backups Gone Stale",
`${stale.length} guest${stale.length !== 1 ? "s have" : " has"} no recent backup:\n\n${stale.map((s) => s.message).join("\n")}`,
{ category: "backup", severity: "warning" },
);
}
@@ -320,6 +445,7 @@ export async function notifyUpdateRunFinished(run: { serverName: string; templat
await notify(
run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`,
`"${run.templateName}" on ${run.serverName}: ${run.message}`,
{ category: "updates", severity: run.ok ? "warning" : "critical" },
);
}
@@ -336,28 +462,32 @@ export async function notifyPbsVerificationFailed(
await notify(
"Homelab Manager — Proxmox Backup Server Verification Failed",
`${failures.length} datastore${failures.length !== 1 ? "s have" : " has"} a problem:\n\n${lines.join("\n")}`,
{ category: "backup", severity: "warning" },
);
}
/** What a problem is about, for routing: its key says the kind and the server; the rest follows its severity. */
const issueMeta = (i: { key?: string; severity?: string }): NotifyMeta => (i.key ? metaOfConditionKey(i.key, i.severity) : { category: null, severity: i.severity === "critical" ? "critical" : "warning" });
export async function notifyHealthIssues(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
if (!(await eventEnabled("healthAlerts"))) return;
await notify(
"Homelab Manager — Health Alert",
`${issues.length} new issue${issues.length !== 1 ? "s" : ""}:\n\n${issues.map((i) => `⚠ ${i.message}`).join("\n")}`,
);
await notifyGrouped(issues, issueMeta, (group) => ({
title: "Homelab Manager — Health Alert",
message: `${group.length} new issue${group.length !== 1 ? "s" : ""}:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`,
}));
}
export async function notifyHealthRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
export async function notifyHealthRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
if (!(await eventEnabled("healthAlerts"))) return;
await notify(
"Homelab Manager — Health Recovered",
`${resolved.length} issue${resolved.length !== 1 ? "s" : ""} cleared:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
);
await notifyGrouped(resolved, issueMeta, (group) => ({
title: "Homelab Manager — Health Recovered",
message: `${group.length} issue${group.length !== 1 ? "s" : ""} cleared:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
}));
}
export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
@@ -365,20 +495,20 @@ export async function notifyAutomationFailed(issues: { message: string; key?: st
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
if (!(await eventEnabled("automationAlerts"))) return;
await notify(
"Homelab Manager — Automation Failed",
`${issues.length} run${issues.length !== 1 ? "s" : ""} failed:\n\n${issues.map((i) => `✕ ${i.message}`).join("\n")}`,
);
await notifyGrouped(issues, issueMeta, (group) => ({
title: "Homelab Manager — Automation Failed",
message: `${group.length} run${group.length !== 1 ? "s" : ""} failed:\n\n${group.map((i) => `✕ ${i.message}`).join("\n")}`,
}));
}
export async function notifyAutomationRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
export async function notifyAutomationRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
if (!(await eventEnabled("automationAlerts"))) return;
await notify(
"Homelab Manager — Automation Recovered",
`${resolved.length} no longer failing:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
);
await notifyGrouped(resolved, issueMeta, (group) => ({
title: "Homelab Manager — Automation Recovered",
message: `${group.length} no longer failing:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
}));
}
export async function notifyDomainExpiry(
@@ -403,7 +533,7 @@ export async function notifyDomainExpiry(
`Couldn't refresh the registration for ${staleChecks.length} domain${staleChecks.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
);
}
await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"));
await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"), { category: "expiry", severity: expiring.some((d) => d.status === "expired") ? "critical" : "warning" });
}
export async function notifyTailscaleKeyExpiry(
@@ -417,6 +547,7 @@ export async function notifyTailscaleKeyExpiry(
await notify(
"Homelab Manager — Tailscale Key Expiry",
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
{ category: "expiry", severity: expiring.some((d) => d.daysLeft < 0) ? "critical" : "warning" },
);
}
@@ -450,11 +581,12 @@ export async function notifyIntegrationDown(source: string, consecutiveFailures:
await notify(
"Homelab Manager — Integration Down",
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
{ category: "integration", severity: "warning" },
);
}
export async function notifyIntegrationRecovered(source: string): Promise<void> {
emitEvent("alert.resolved", { alert: { key: `integration:${source}`, message: `${sourceLabel(source)} is answering again.` } });
if (!(await eventEnabled("integrationFailureAlerts"))) return;
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`, { category: "integration", severity: "warning" });
}
+114
View File
@@ -0,0 +1,114 @@
/**
* Who hears what. By default every notification goes to every channel that is switched on. Routing rules narrow that: the first rule a
* notification matches decides which channels get it (possibly none — it is then only on the Alerts page), and each browser that has asked
* for push notifications may further say it only wants some severities or kinds. This file is the pure part: what a notification is about,
* which rule matches and what a browser accepts. Sending is in notify.ts.
*/
import type { AlertCategory, AlertSeverity } from "./alertTypes.js";
export const CHANNEL_IDS = ["gotify", "ntfy", "smtp", "webhook", "webPush"] as const;
export type ChannelId = (typeof CHANNEL_IDS)[number];
export const CATEGORIES = ["offline", "disk", "services", "backup", "updates", "expiry", "automation", "integration", "monitoring", "tickets"] as const satisfies readonly AlertCategory[];
/** What a notification is about, as far as routing cares. Anything left out matches only rules that don't ask about it. */
export interface NotifyMeta {
category?: AlertCategory | null;
severity?: AlertSeverity;
/** The server it is about, whose tags a rule may ask for. */
serverId?: number | null;
/** Filled in from the server before matching. */
tags?: string[];
}
export interface RoutingRule {
id: string;
name: string;
enabled: boolean;
/** Empty = any kind. */
categories: AlertCategory[];
/** null = any severity. */
minSeverity: "warning" | "critical" | null;
/** Empty = any server. Otherwise the notification must be about a server that has at least one of these tags. */
tags: string[];
/** The channels that get it. Empty = none (the Alerts page still shows it). */
channels: ChannelId[];
}
export interface RoutingSettings {
rules: RoutingRule[];
}
export interface EscalationStep {
/** Minutes an alert must have been raised, with nobody on it, before this step fires. */
afterMinutes: number;
minSeverity: "warning" | "critical";
channels: ChannelId[];
/** An address to email as well (someone else), through the email channel's server. Blank = none. */
extraEmail: string;
/** Send even during quiet hours — the point of an escalation can be to wake someone. */
evenInQuietHours: boolean;
}
export interface EscalationSettings {
enabled: boolean;
steps: EscalationStep[];
}
const RANK: Record<string, number> = { info: 0, warning: 1, critical: 2 };
export const severityRank = (s: string | undefined | null) => RANK[s ?? "warning"] ?? 1;
/** Whether a rule applies to a notification. A rule that asks about something the notification doesn't say doesn't match. */
export function ruleMatches(rule: RoutingRule, meta: NotifyMeta): boolean {
if (!rule.enabled) return false;
if (rule.categories.length > 0 && !(meta.category && rule.categories.includes(meta.category))) return false;
if (rule.minSeverity && severityRank(meta.severity) < severityRank(rule.minSeverity)) return false;
if (rule.tags.length > 0) {
const have = new Set((meta.tags ?? []).map((t) => t.toLowerCase()));
if (!rule.tags.some((t) => have.has(t.toLowerCase()))) return false;
}
return true;
}
/** The first rule that applies, or null (every channel that is on). */
export function firstMatchingRule(rules: RoutingRule[], meta: NotifyMeta): RoutingRule | null {
return rules.find((r) => ruleMatches(r, meta)) ?? null;
}
/** A browser's own wishes, as stored. */
export interface PushPrefs {
minSeverity: string | null;
categories: string | null;
}
export function parseCategories(stored: string | null): AlertCategory[] | null {
if (!stored) return null;
try {
const v = JSON.parse(stored);
if (!Array.isArray(v)) return null;
const list = v.filter((c): c is AlertCategory => (CATEGORIES as readonly string[]).includes(c));
return list.length > 0 ? list : null;
} catch {
return null;
}
}
/** Whether a browser wants this notification. What the notification doesn't say can't exclude it. */
export function pushAccepts(prefs: PushPrefs, meta: NotifyMeta | undefined): boolean {
if (!meta) return true;
if (prefs.minSeverity && meta.severity && severityRank(meta.severity) < severityRank(prefs.minSeverity)) return false;
const wanted = parseCategories(prefs.categories);
if (wanted && meta.category && !wanted.includes(meta.category)) return false;
return true;
}
/** What kind of problem a health or automation condition key is about, and which server (for its tags). */
export function metaOfConditionKey(key: string, severity?: string): NotifyMeta {
let category: AlertCategory | null = null;
if (key.startsWith("offline:")) category = "offline";
else if (key.startsWith("failed-units:")) category = "services";
else if (/^(disk|smart|zfs|synology-volume|synology-disk):/.test(key)) category = "disk";
else if (key.startsWith("automation:")) category = "automation";
const server = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)/.exec(key);
return { category, severity: severity === "critical" ? "critical" : "warning", serverId: server ? Number(server[1]) : null };
}
+9
View File
@@ -2,6 +2,7 @@ import { sql } from "drizzle-orm";
import { db } from "../db/client.js";
import { settings } from "../db/schema.js";
import { defaultThemes, type NameTheme } from "./nameThemes.js";
import type { EscalationSettings, RoutingSettings } from "./notifyRouting.js";
import { env } from "../env.js";
import { decryptSecret, encryptSecret } from "../crypto.js";
@@ -44,6 +45,8 @@ export interface WebPushSettings {
contact: string;
}
export type { RoutingSettings, EscalationSettings } from "./notifyRouting.js";
export interface NotificationEvents {
dnsAdd: boolean;
dnsUpdate: boolean;
@@ -197,6 +200,10 @@ export interface AppSettings {
smtp: SmtpSettings;
webhook: WebhookSettings;
webPush: WebPushSettings;
/** Which channels hear which notifications. */
routing: RoutingSettings;
/** Re-notifying about alerts nobody has acknowledged. */
escalation: EscalationSettings;
notifications: NotificationEvents;
providerColors: ProviderColors;
integrationColors: IntegrationColors;
@@ -220,6 +227,8 @@ const DEFAULTS: AppSettings = {
smtp: { enabled: false, host: "", port: 587, secure: false, username: "", password: "", from: "", to: "" },
webhook: { enabled: false, url: "", secret: "" },
webPush: { enabled: false, contact: "" },
routing: { rules: [] },
escalation: { enabled: false, steps: [] },
notifications: {
dnsAdd: true,
dnsUpdate: true,