Notification routing rules, per-browser push wishes, escalation of unacknowledged alerts

- Routing: rules pick channels by kind, severity and server tags (first match wins, none = nobody); mixed batches split per route; recoveries follow their alert; quiet-hours digest keeps each notification's audience
- Push: each subscribed browser can ask for a minimum severity and some kinds
- Escalation: up to three timed steps re-notify alerts nobody has acknowledged, through chosen channels and an optional extra email; an acknowledgement stops them
- Fix ntfy: titles with an em dash made every ntfy send fail (header not Latin-1); now RFC 2047 encoded

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 19:44:45 +02:00
1 parent 9caa414829
commit 43fc5e0272
21 files changed
+4769 -55

No files matched your search

+170 -38
View File
@@ -3,6 +3,11 @@ import { getSettings, type NotificationEvents } from "./settingsStore.js";
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
import { emitEvent } from "./eventWebhooks.js";
import { sendPush, subscriptionCount } from "./webPush.js";
import { db } from "../db/client.js";
import { inArray } from "drizzle-orm";
import { pushSubscriptions, servers } from "../db/schema.js";
import { parseTags } from "./serverTags.js";
import { firstMatchingRule, metaOfConditionKey, pushAccepts, type ChannelId, type NotifyMeta } from "./notifyRouting.js";
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
@@ -22,13 +27,22 @@ async function sendGotify(title: string, message: string) {
}
}
/**
* A header can only carry Latin-1, and every title here has an em dash. ntfy reads RFC 2047 encoded words in its headers ("=?UTF-8?B?...?="),
* so anything outside plain ASCII is sent that way.
*/
export function headerSafe(text: string): string {
// eslint-disable-next-line no-control-regex
return /^[ -~]*$/.test(text) ? text : `=?UTF-8?B?${Buffer.from(text, "utf8").toString("base64")}?=`;
}
async function sendNtfy(title: string, message: string) {
const { ntfy } = await getSettings();
if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return;
const base = ntfy.url.replace(/\/$/, "");
const headers: Record<string, string> = {
"Content-Type": "text/plain",
Title: title,
Title: headerSafe(title),
Priority: String(ntfy.priority ?? 3),
};
if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`;
@@ -73,18 +87,116 @@ async function sendWebhook(title: string, message: string) {
}
}
async function sendWebPush(title: string, message: string) {
async function sendWebPush(title: string, message: string, only: number[] | null) {
const { webPush } = await getSettings();
if (!webPush.enabled) return;
try {
await sendPush({ title, body: message });
if (only === null) await sendPush({ title, body: message });
else if (only.length > 0) await sendPush({ title, body: message }, await db.select().from(pushSubscriptions).where(inArray(pushSubscriptions.id, only)));
} catch (err) {
console.error("[notify] Web push failed:", err instanceof Error ? err.message : err);
}
}
async function sendToChannels(title: string, message: string): Promise<void> {
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message), sendWebPush(title, message)]);
/** Email to one address, through the email channel's server, whatever the channel's own recipient is (an escalation to someone else). */
async function sendExtraEmail(title: string, message: string, to: string) {
const { smtp } = await getSettings();
if (!smtp.host || !smtp.from) {
console.error("[notify] An escalation wants to email someone, but email isn't set up.");
return;
}
try {
const transporter = nodemailer.createTransport({
host: smtp.host,
port: Number(smtp.port) || 587,
secure: !!smtp.secure,
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
});
await transporter.sendMail({ from: smtp.from, to, subject: title, text: message });
} catch (err) {
console.error("[notify] Extra email failed:", err instanceof Error ? err.message : err);
}
}
// ─── Who a notification is for ─────────────────────────────────────────────
/** The channels and browsers a notification goes to. `null` means every one that is on (the way it was before routing existed). */
export interface Audience {
channels: ChannelId[] | null;
/** The subscribed browsers that want it; null = all of them. */
pushIds: number[] | null;
/** Also email this address. */
email?: string;
}
export const EVERYONE: Audience = { channels: null, pushIds: null };
export const audienceKey = (a: Audience) => JSON.stringify({ c: a.channels ? [...a.channels].sort() : null, p: a.pushIds ? [...a.pushIds].sort((x, y) => x - y) : null, e: a.email ?? null });
/** Told to nobody: a rule said so (the Alerts page still shows it). */
const isMuted = (a: Audience) => a.channels !== null && a.channels.length === 0 && !a.email;
/** Works out who a notification is for: the first routing rule it matches picks the channels, and browsers may turn down what they don't want. */
export async function audienceFor(meta?: NotifyMeta): Promise<Audience> {
if (!meta) return EVERYONE;
const { routing } = await getSettings();
let channels: ChannelId[] | null = null;
if (routing.rules.some((r) => r.enabled)) {
let tags = meta.tags;
if (!tags && meta.serverId != null) {
const [row] = await db.select({ tags: servers.tags }).from(servers).where(inArray(servers.id, [meta.serverId]));
tags = parseTags(row?.tags);
}
channels = firstMatchingRule(routing.rules, { ...meta, tags: tags ?? [] })?.channels ?? null;
}
let pushIds: number[] | null = null;
if (channels === null || channels.includes("webPush")) {
const subs = await db.select({ id: pushSubscriptions.id, minSeverity: pushSubscriptions.minSeverity, categories: pushSubscriptions.categories }).from(pushSubscriptions);
// Only when somebody has said what they want; otherwise everyone gets it, as ever.
if (subs.some((s) => s.minSeverity || s.categories)) pushIds = subs.filter((s) => pushAccepts(s, meta)).map((s) => s.id);
}
return { channels, pushIds };
}
async function sendToChannels(title: string, message: string, audience: Audience = EVERYONE): Promise<void> {
const to = (c: ChannelId) => audience.channels === null || audience.channels.includes(c);
await Promise.all([
to("gotify") ? sendGotify(title, message) : undefined,
to("ntfy") ? sendNtfy(title, message) : undefined,
to("smtp") ? sendSmtp(title, message) : undefined,
to("webhook") ? sendWebhook(title, message) : undefined,
to("webPush") ? sendWebPush(title, message, audience.pushIds) : undefined,
audience.email ? sendExtraEmail(title, message, audience.email) : undefined,
]);
}
/**
* Sends to an audience that is already worked out: held for the quiet-hours digest if it is quiet (unless told not to), nowhere if a rule
* said nobody. Escalations use this with their own channels.
*/
export async function notifyAudience(title: string, message: string, audience: Audience, options: { quietHours?: boolean } = {}): Promise<void> {
if (isMuted(audience)) return;
if (options.quietHours !== false && (await isQuietHoursActive())) {
await enqueueNotification(title, message, audience.channels === null && audience.pushIds === null && !audience.email ? null : JSON.stringify(audience));
return;
}
await sendToChannels(title, message, audience);
}
/** Items that may each be for a different audience (a rule can send a disk warning one place and a server outage another): sent as one message per audience. */
async function notifyGrouped<T>(items: T[], metaOf: (item: T) => NotifyMeta, build: (group: T[]) => { title: string; message: string }): Promise<void> {
const groups = new Map<string, { audience: Audience; items: T[] }>();
for (const item of items) {
const audience = await audienceFor(metaOf(item));
const key = audienceKey(audience);
const group = groups.get(key) ?? { audience, items: [] };
group.items.push(item);
groups.set(key, group);
}
for (const group of groups.values()) {
const { title, message } = build(group.items);
await notifyAudience(title, message, group.audience);
}
}
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
@@ -133,23 +245,32 @@ export async function enabledChannelCount(): Promise<number> {
].filter(Boolean).length;
}
export async function notify(title: string, message: string): Promise<void> {
if (await isQuietHoursActive()) {
await enqueueNotification(title, message);
return;
}
await sendToChannels(title, message);
/** `meta` says what it is about, so routing rules and browsers' wishes can apply; without it, it goes to everyone. */
export async function notify(title: string, message: string, meta?: NotifyMeta): Promise<void> {
await notifyAudience(title, message, await audienceFor(meta));
}
/** Called by the daily scheduler at quiet-hours end (or manually) — sends everything queued during the window as one digest, then clears it. */
export async function flushQuietHoursQueue(): Promise<number> {
const queued = await listQueuedNotifications();
if (queued.length === 0) return 0;
const sections = queued.map((q) => `— ${q.title} —\n${q.message}`);
await sendToChannels(
"Homelab Manager — Notifications from quiet hours",
`${queued.length} notification${queued.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
);
// One digest for each audience the held notifications were for.
const groups = new Map<string, typeof queued>();
for (const q of queued) groups.set(q.audience ?? "", [...(groups.get(q.audience ?? "") ?? []), q]);
for (const [stored, items] of groups) {
let audience: Audience = EVERYONE;
try {
if (stored) audience = JSON.parse(stored) as Audience;
} catch {
// unreadable: everyone, as before
}
const sections = items.map((q) => `— ${q.title} —\n${q.message}`);
await sendToChannels(
"Homelab Manager — Notifications from quiet hours",
`${items.length} notification${items.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
audience,
);
}
await clearQueuedNotifications();
return queued.length;
}
@@ -172,7 +293,7 @@ export async function testGotify(cfg: { url: string; token: string; priority?: n
export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise<void> {
const base = cfg.url.replace(/\/$/, "");
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: "Homelab Manager — Test", Priority: String(cfg.priority ?? 3) };
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: headerSafe("Homelab Manager — Test"), Priority: String(cfg.priority ?? 3) };
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, {
method: "POST",
@@ -263,7 +384,7 @@ export async function notifySecretExpiry(
`Couldn't read the live certificate for ${checkFailures.length} monitored secret${checkFailures.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
);
}
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"));
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"), { category: "expiry", severity: expiring.some((s) => s.status === "expired") ? "critical" : "warning" });
}
export async function notifyDockerUpdates(
@@ -277,6 +398,7 @@ export async function notifyDockerUpdates(
await notify(
"Homelab Manager — Docker Updates Available",
`${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`,
{ category: "updates", severity: "warning" },
);
}
@@ -291,6 +413,7 @@ export async function notifyProxmoxBackupFailure(
await notify(
"Homelab Manager — Proxmox Backup Failed",
`${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
{ category: "backup", severity: "warning" },
);
}
@@ -303,6 +426,7 @@ export async function notifyProxmoxUncoveredGuests(
await notify(
"Homelab Manager — Proxmox Guests Without Backup Coverage",
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
{ category: "backup", severity: "warning" },
);
}
@@ -312,6 +436,7 @@ export async function notifyStaleBackups(stale: { message: string }[]): Promise<
await notify(
"Homelab Manager — Backups Gone Stale",
`${stale.length} guest${stale.length !== 1 ? "s have" : " has"} no recent backup:\n\n${stale.map((s) => s.message).join("\n")}`,
{ category: "backup", severity: "warning" },
);
}
@@ -320,6 +445,7 @@ export async function notifyUpdateRunFinished(run: { serverName: string; templat
await notify(
run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`,
`"${run.templateName}" on ${run.serverName}: ${run.message}`,
{ category: "updates", severity: run.ok ? "warning" : "critical" },
);
}
@@ -336,28 +462,32 @@ export async function notifyPbsVerificationFailed(
await notify(
"Homelab Manager — Proxmox Backup Server Verification Failed",
`${failures.length} datastore${failures.length !== 1 ? "s have" : " has"} a problem:\n\n${lines.join("\n")}`,
{ category: "backup", severity: "warning" },
);
}
/** What a problem is about, for routing: its key says the kind and the server; the rest follows its severity. */
const issueMeta = (i: { key?: string; severity?: string }): NotifyMeta => (i.key ? metaOfConditionKey(i.key, i.severity) : { category: null, severity: i.severity === "critical" ? "critical" : "warning" });
export async function notifyHealthIssues(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
if (!(await eventEnabled("healthAlerts"))) return;
await notify(
"Homelab Manager — Health Alert",
`${issues.length} new issue${issues.length !== 1 ? "s" : ""}:\n\n${issues.map((i) => `⚠ ${i.message}`).join("\n")}`,
);
await notifyGrouped(issues, issueMeta, (group) => ({
title: "Homelab Manager — Health Alert",
message: `${group.length} new issue${group.length !== 1 ? "s" : ""}:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`,
}));
}
export async function notifyHealthRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
export async function notifyHealthRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
if (!(await eventEnabled("healthAlerts"))) return;
await notify(
"Homelab Manager — Health Recovered",
`${resolved.length} issue${resolved.length !== 1 ? "s" : ""} cleared:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
);
await notifyGrouped(resolved, issueMeta, (group) => ({
title: "Homelab Manager — Health Recovered",
message: `${group.length} issue${group.length !== 1 ? "s" : ""} cleared:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
}));
}
export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
@@ -365,20 +495,20 @@ export async function notifyAutomationFailed(issues: { message: string; key?: st
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
if (!(await eventEnabled("automationAlerts"))) return;
await notify(
"Homelab Manager — Automation Failed",
`${issues.length} run${issues.length !== 1 ? "s" : ""} failed:\n\n${issues.map((i) => `✕ ${i.message}`).join("\n")}`,
);
await notifyGrouped(issues, issueMeta, (group) => ({
title: "Homelab Manager — Automation Failed",
message: `${group.length} run${group.length !== 1 ? "s" : ""} failed:\n\n${group.map((i) => `✕ ${i.message}`).join("\n")}`,
}));
}
export async function notifyAutomationRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
export async function notifyAutomationRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
if (!(await eventEnabled("automationAlerts"))) return;
await notify(
"Homelab Manager — Automation Recovered",
`${resolved.length} no longer failing:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
);
await notifyGrouped(resolved, issueMeta, (group) => ({
title: "Homelab Manager — Automation Recovered",
message: `${group.length} no longer failing:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
}));
}
export async function notifyDomainExpiry(
@@ -403,7 +533,7 @@ export async function notifyDomainExpiry(
`Couldn't refresh the registration for ${staleChecks.length} domain${staleChecks.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
);
}
await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"));
await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"), { category: "expiry", severity: expiring.some((d) => d.status === "expired") ? "critical" : "warning" });
}
export async function notifyTailscaleKeyExpiry(
@@ -417,6 +547,7 @@ export async function notifyTailscaleKeyExpiry(
await notify(
"Homelab Manager — Tailscale Key Expiry",
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
{ category: "expiry", severity: expiring.some((d) => d.daysLeft < 0) ? "critical" : "warning" },
);
}
@@ -450,11 +581,12 @@ export async function notifyIntegrationDown(source: string, consecutiveFailures:
await notify(
"Homelab Manager — Integration Down",
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
{ category: "integration", severity: "warning" },
);
}
export async function notifyIntegrationRecovered(source: string): Promise<void> {
emitEvent("alert.resolved", { alert: { key: `integration:${source}`, message: `${sourceLabel(source)} is answering again.` } });
if (!(await eventEnabled("integrationFailureAlerts"))) return;
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`, { category: "integration", severity: "warning" });
}