Notification routing rules, per-browser push wishes, escalation of unacknowledged alerts
- Routing: rules pick channels by kind, severity and server tags (first match wins, none = nobody); mixed batches split per route; recoveries follow their alert; quiet-hours digest keeps each notification's audience - Push: each subscribed browser can ask for a minimum severity and some kinds - Escalation: up to three timed steps re-notify alerts nobody has acknowledged, through chosen channels and an optional extra email; an acknowledgement stops them - Fix ntfy: titles with an em dash made every ntfy send fail (header not Latin-1); now RFC 2047 encoded Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9caa414829
commit
43fc5e0272
21 files changed
+4769
-55
No files matched your search
+170
-38
@@ -3,6 +3,11 @@ import { getSettings, type NotificationEvents } from "./settingsStore.js";
|
||||
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
|
||||
import { emitEvent } from "./eventWebhooks.js";
|
||||
import { sendPush, subscriptionCount } from "./webPush.js";
|
||||
import { db } from "../db/client.js";
|
||||
import { inArray } from "drizzle-orm";
|
||||
import { pushSubscriptions, servers } from "../db/schema.js";
|
||||
import { parseTags } from "./serverTags.js";
|
||||
import { firstMatchingRule, metaOfConditionKey, pushAccepts, type ChannelId, type NotifyMeta } from "./notifyRouting.js";
|
||||
|
||||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||||
|
||||
@@ -22,13 +27,22 @@ async function sendGotify(title: string, message: string) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A header can only carry Latin-1, and every title here has an em dash. ntfy reads RFC 2047 encoded words in its headers ("=?UTF-8?B?...?="),
|
||||
* so anything outside plain ASCII is sent that way.
|
||||
*/
|
||||
export function headerSafe(text: string): string {
|
||||
// eslint-disable-next-line no-control-regex
|
||||
return /^[ -~]*$/.test(text) ? text : `=?UTF-8?B?${Buffer.from(text, "utf8").toString("base64")}?=`;
|
||||
}
|
||||
|
||||
async function sendNtfy(title: string, message: string) {
|
||||
const { ntfy } = await getSettings();
|
||||
if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return;
|
||||
const base = ntfy.url.replace(/\/$/, "");
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "text/plain",
|
||||
Title: title,
|
||||
Title: headerSafe(title),
|
||||
Priority: String(ntfy.priority ?? 3),
|
||||
};
|
||||
if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`;
|
||||
@@ -73,18 +87,116 @@ async function sendWebhook(title: string, message: string) {
|
||||
}
|
||||
}
|
||||
|
||||
async function sendWebPush(title: string, message: string) {
|
||||
async function sendWebPush(title: string, message: string, only: number[] | null) {
|
||||
const { webPush } = await getSettings();
|
||||
if (!webPush.enabled) return;
|
||||
try {
|
||||
await sendPush({ title, body: message });
|
||||
if (only === null) await sendPush({ title, body: message });
|
||||
else if (only.length > 0) await sendPush({ title, body: message }, await db.select().from(pushSubscriptions).where(inArray(pushSubscriptions.id, only)));
|
||||
} catch (err) {
|
||||
console.error("[notify] Web push failed:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
}
|
||||
|
||||
async function sendToChannels(title: string, message: string): Promise<void> {
|
||||
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message), sendWebPush(title, message)]);
|
||||
/** Email to one address, through the email channel's server, whatever the channel's own recipient is (an escalation to someone else). */
|
||||
async function sendExtraEmail(title: string, message: string, to: string) {
|
||||
const { smtp } = await getSettings();
|
||||
if (!smtp.host || !smtp.from) {
|
||||
console.error("[notify] An escalation wants to email someone, but email isn't set up.");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: smtp.host,
|
||||
port: Number(smtp.port) || 587,
|
||||
secure: !!smtp.secure,
|
||||
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
|
||||
});
|
||||
await transporter.sendMail({ from: smtp.from, to, subject: title, text: message });
|
||||
} catch (err) {
|
||||
console.error("[notify] Extra email failed:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Who a notification is for ─────────────────────────────────────────────
|
||||
|
||||
/** The channels and browsers a notification goes to. `null` means every one that is on (the way it was before routing existed). */
|
||||
export interface Audience {
|
||||
channels: ChannelId[] | null;
|
||||
/** The subscribed browsers that want it; null = all of them. */
|
||||
pushIds: number[] | null;
|
||||
/** Also email this address. */
|
||||
email?: string;
|
||||
}
|
||||
|
||||
export const EVERYONE: Audience = { channels: null, pushIds: null };
|
||||
|
||||
export const audienceKey = (a: Audience) => JSON.stringify({ c: a.channels ? [...a.channels].sort() : null, p: a.pushIds ? [...a.pushIds].sort((x, y) => x - y) : null, e: a.email ?? null });
|
||||
|
||||
/** Told to nobody: a rule said so (the Alerts page still shows it). */
|
||||
const isMuted = (a: Audience) => a.channels !== null && a.channels.length === 0 && !a.email;
|
||||
|
||||
/** Works out who a notification is for: the first routing rule it matches picks the channels, and browsers may turn down what they don't want. */
|
||||
export async function audienceFor(meta?: NotifyMeta): Promise<Audience> {
|
||||
if (!meta) return EVERYONE;
|
||||
const { routing } = await getSettings();
|
||||
let channels: ChannelId[] | null = null;
|
||||
if (routing.rules.some((r) => r.enabled)) {
|
||||
let tags = meta.tags;
|
||||
if (!tags && meta.serverId != null) {
|
||||
const [row] = await db.select({ tags: servers.tags }).from(servers).where(inArray(servers.id, [meta.serverId]));
|
||||
tags = parseTags(row?.tags);
|
||||
}
|
||||
channels = firstMatchingRule(routing.rules, { ...meta, tags: tags ?? [] })?.channels ?? null;
|
||||
}
|
||||
let pushIds: number[] | null = null;
|
||||
if (channels === null || channels.includes("webPush")) {
|
||||
const subs = await db.select({ id: pushSubscriptions.id, minSeverity: pushSubscriptions.minSeverity, categories: pushSubscriptions.categories }).from(pushSubscriptions);
|
||||
// Only when somebody has said what they want; otherwise everyone gets it, as ever.
|
||||
if (subs.some((s) => s.minSeverity || s.categories)) pushIds = subs.filter((s) => pushAccepts(s, meta)).map((s) => s.id);
|
||||
}
|
||||
return { channels, pushIds };
|
||||
}
|
||||
|
||||
async function sendToChannels(title: string, message: string, audience: Audience = EVERYONE): Promise<void> {
|
||||
const to = (c: ChannelId) => audience.channels === null || audience.channels.includes(c);
|
||||
await Promise.all([
|
||||
to("gotify") ? sendGotify(title, message) : undefined,
|
||||
to("ntfy") ? sendNtfy(title, message) : undefined,
|
||||
to("smtp") ? sendSmtp(title, message) : undefined,
|
||||
to("webhook") ? sendWebhook(title, message) : undefined,
|
||||
to("webPush") ? sendWebPush(title, message, audience.pushIds) : undefined,
|
||||
audience.email ? sendExtraEmail(title, message, audience.email) : undefined,
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends to an audience that is already worked out: held for the quiet-hours digest if it is quiet (unless told not to), nowhere if a rule
|
||||
* said nobody. Escalations use this with their own channels.
|
||||
*/
|
||||
export async function notifyAudience(title: string, message: string, audience: Audience, options: { quietHours?: boolean } = {}): Promise<void> {
|
||||
if (isMuted(audience)) return;
|
||||
if (options.quietHours !== false && (await isQuietHoursActive())) {
|
||||
await enqueueNotification(title, message, audience.channels === null && audience.pushIds === null && !audience.email ? null : JSON.stringify(audience));
|
||||
return;
|
||||
}
|
||||
await sendToChannels(title, message, audience);
|
||||
}
|
||||
|
||||
/** Items that may each be for a different audience (a rule can send a disk warning one place and a server outage another): sent as one message per audience. */
|
||||
async function notifyGrouped<T>(items: T[], metaOf: (item: T) => NotifyMeta, build: (group: T[]) => { title: string; message: string }): Promise<void> {
|
||||
const groups = new Map<string, { audience: Audience; items: T[] }>();
|
||||
for (const item of items) {
|
||||
const audience = await audienceFor(metaOf(item));
|
||||
const key = audienceKey(audience);
|
||||
const group = groups.get(key) ?? { audience, items: [] };
|
||||
group.items.push(item);
|
||||
groups.set(key, group);
|
||||
}
|
||||
for (const group of groups.values()) {
|
||||
const { title, message } = build(group.items);
|
||||
await notifyAudience(title, message, group.audience);
|
||||
}
|
||||
}
|
||||
|
||||
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
|
||||
@@ -133,23 +245,32 @@ export async function enabledChannelCount(): Promise<number> {
|
||||
].filter(Boolean).length;
|
||||
}
|
||||
|
||||
export async function notify(title: string, message: string): Promise<void> {
|
||||
if (await isQuietHoursActive()) {
|
||||
await enqueueNotification(title, message);
|
||||
return;
|
||||
}
|
||||
await sendToChannels(title, message);
|
||||
/** `meta` says what it is about, so routing rules and browsers' wishes can apply; without it, it goes to everyone. */
|
||||
export async function notify(title: string, message: string, meta?: NotifyMeta): Promise<void> {
|
||||
await notifyAudience(title, message, await audienceFor(meta));
|
||||
}
|
||||
|
||||
/** Called by the daily scheduler at quiet-hours end (or manually) — sends everything queued during the window as one digest, then clears it. */
|
||||
export async function flushQuietHoursQueue(): Promise<number> {
|
||||
const queued = await listQueuedNotifications();
|
||||
if (queued.length === 0) return 0;
|
||||
const sections = queued.map((q) => `— ${q.title} —\n${q.message}`);
|
||||
await sendToChannels(
|
||||
"Homelab Manager — Notifications from quiet hours",
|
||||
`${queued.length} notification${queued.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
|
||||
);
|
||||
// One digest for each audience the held notifications were for.
|
||||
const groups = new Map<string, typeof queued>();
|
||||
for (const q of queued) groups.set(q.audience ?? "", [...(groups.get(q.audience ?? "") ?? []), q]);
|
||||
for (const [stored, items] of groups) {
|
||||
let audience: Audience = EVERYONE;
|
||||
try {
|
||||
if (stored) audience = JSON.parse(stored) as Audience;
|
||||
} catch {
|
||||
// unreadable: everyone, as before
|
||||
}
|
||||
const sections = items.map((q) => `— ${q.title} —\n${q.message}`);
|
||||
await sendToChannels(
|
||||
"Homelab Manager — Notifications from quiet hours",
|
||||
`${items.length} notification${items.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
|
||||
audience,
|
||||
);
|
||||
}
|
||||
await clearQueuedNotifications();
|
||||
return queued.length;
|
||||
}
|
||||
@@ -172,7 +293,7 @@ export async function testGotify(cfg: { url: string; token: string; priority?: n
|
||||
|
||||
export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise<void> {
|
||||
const base = cfg.url.replace(/\/$/, "");
|
||||
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: "Homelab Manager — Test", Priority: String(cfg.priority ?? 3) };
|
||||
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: headerSafe("Homelab Manager — Test"), Priority: String(cfg.priority ?? 3) };
|
||||
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
|
||||
const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, {
|
||||
method: "POST",
|
||||
@@ -263,7 +384,7 @@ export async function notifySecretExpiry(
|
||||
`Couldn't read the live certificate for ${checkFailures.length} monitored secret${checkFailures.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
|
||||
);
|
||||
}
|
||||
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"));
|
||||
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"), { category: "expiry", severity: expiring.some((s) => s.status === "expired") ? "critical" : "warning" });
|
||||
}
|
||||
|
||||
export async function notifyDockerUpdates(
|
||||
@@ -277,6 +398,7 @@ export async function notifyDockerUpdates(
|
||||
await notify(
|
||||
"Homelab Manager — Docker Updates Available",
|
||||
`${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`,
|
||||
{ category: "updates", severity: "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -291,6 +413,7 @@ export async function notifyProxmoxBackupFailure(
|
||||
await notify(
|
||||
"Homelab Manager — Proxmox Backup Failed",
|
||||
`${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
|
||||
{ category: "backup", severity: "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -303,6 +426,7 @@ export async function notifyProxmoxUncoveredGuests(
|
||||
await notify(
|
||||
"Homelab Manager — Proxmox Guests Without Backup Coverage",
|
||||
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
|
||||
{ category: "backup", severity: "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -312,6 +436,7 @@ export async function notifyStaleBackups(stale: { message: string }[]): Promise<
|
||||
await notify(
|
||||
"Homelab Manager — Backups Gone Stale",
|
||||
`${stale.length} guest${stale.length !== 1 ? "s have" : " has"} no recent backup:\n\n${stale.map((s) => s.message).join("\n")}`,
|
||||
{ category: "backup", severity: "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -320,6 +445,7 @@ export async function notifyUpdateRunFinished(run: { serverName: string; templat
|
||||
await notify(
|
||||
run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`,
|
||||
`"${run.templateName}" on ${run.serverName}: ${run.message}`,
|
||||
{ category: "updates", severity: run.ok ? "warning" : "critical" },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -336,28 +462,32 @@ export async function notifyPbsVerificationFailed(
|
||||
await notify(
|
||||
"Homelab Manager — Proxmox Backup Server Verification Failed",
|
||||
`${failures.length} datastore${failures.length !== 1 ? "s have" : " has"} a problem:\n\n${lines.join("\n")}`,
|
||||
{ category: "backup", severity: "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
/** What a problem is about, for routing: its key says the kind and the server; the rest follows its severity. */
|
||||
const issueMeta = (i: { key?: string; severity?: string }): NotifyMeta => (i.key ? metaOfConditionKey(i.key, i.severity) : { category: null, severity: i.severity === "critical" ? "critical" : "warning" });
|
||||
|
||||
export async function notifyHealthIssues(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
|
||||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||||
if (issues.length === 0) return;
|
||||
if (!(await eventEnabled("healthAlerts"))) return;
|
||||
await notify(
|
||||
"Homelab Manager — Health Alert",
|
||||
`${issues.length} new issue${issues.length !== 1 ? "s" : ""}:\n\n${issues.map((i) => `⚠ ${i.message}`).join("\n")}`,
|
||||
);
|
||||
await notifyGrouped(issues, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Health Alert",
|
||||
message: `${group.length} new issue${group.length !== 1 ? "s" : ""}:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function notifyHealthRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
|
||||
export async function notifyHealthRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||||
if (resolved.length === 0) return;
|
||||
if (!(await eventEnabled("healthAlerts"))) return;
|
||||
await notify(
|
||||
"Homelab Manager — Health Recovered",
|
||||
`${resolved.length} issue${resolved.length !== 1 ? "s" : ""} cleared:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||||
);
|
||||
await notifyGrouped(resolved, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Health Recovered",
|
||||
message: `${group.length} issue${group.length !== 1 ? "s" : ""} cleared:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
|
||||
@@ -365,20 +495,20 @@ export async function notifyAutomationFailed(issues: { message: string; key?: st
|
||||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||||
if (issues.length === 0) return;
|
||||
if (!(await eventEnabled("automationAlerts"))) return;
|
||||
await notify(
|
||||
"Homelab Manager — Automation Failed",
|
||||
`${issues.length} run${issues.length !== 1 ? "s" : ""} failed:\n\n${issues.map((i) => `✕ ${i.message}`).join("\n")}`,
|
||||
);
|
||||
await notifyGrouped(issues, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Automation Failed",
|
||||
message: `${group.length} run${group.length !== 1 ? "s" : ""} failed:\n\n${group.map((i) => `✕ ${i.message}`).join("\n")}`,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function notifyAutomationRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
|
||||
export async function notifyAutomationRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||||
if (resolved.length === 0) return;
|
||||
if (!(await eventEnabled("automationAlerts"))) return;
|
||||
await notify(
|
||||
"Homelab Manager — Automation Recovered",
|
||||
`${resolved.length} no longer failing:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||||
);
|
||||
await notifyGrouped(resolved, issueMeta, (group) => ({
|
||||
title: "Homelab Manager — Automation Recovered",
|
||||
message: `${group.length} no longer failing:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function notifyDomainExpiry(
|
||||
@@ -403,7 +533,7 @@ export async function notifyDomainExpiry(
|
||||
`Couldn't refresh the registration for ${staleChecks.length} domain${staleChecks.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
|
||||
);
|
||||
}
|
||||
await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"));
|
||||
await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"), { category: "expiry", severity: expiring.some((d) => d.status === "expired") ? "critical" : "warning" });
|
||||
}
|
||||
|
||||
export async function notifyTailscaleKeyExpiry(
|
||||
@@ -417,6 +547,7 @@ export async function notifyTailscaleKeyExpiry(
|
||||
await notify(
|
||||
"Homelab Manager — Tailscale Key Expiry",
|
||||
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
|
||||
{ category: "expiry", severity: expiring.some((d) => d.daysLeft < 0) ? "critical" : "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -450,11 +581,12 @@ export async function notifyIntegrationDown(source: string, consecutiveFailures:
|
||||
await notify(
|
||||
"Homelab Manager — Integration Down",
|
||||
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
|
||||
{ category: "integration", severity: "warning" },
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyIntegrationRecovered(source: string): Promise<void> {
|
||||
emitEvent("alert.resolved", { alert: { key: `integration:${source}`, message: `${sourceLabel(source)} is answering again.` } });
|
||||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||||
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
|
||||
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`, { category: "integration", severity: "warning" });
|
||||
}
|
||||
Reference in new issue
Block a user