import nodemailer from "nodemailer"; import { getSettings, type NotificationEvents } from "./settingsStore.js"; import { holdFor, holdReason } from "./availability.js"; import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js"; import { emitEvent } from "./eventWebhooks.js"; import { sendPush, subscriptionCount } from "./webPush.js"; import { db } from "../db/client.js"; import { inArray } from "drizzle-orm"; import { pushSubscriptions, servers } from "../db/schema.js"; import { parseTags } from "./serverTags.js"; import { dampRaised, dampRecovered } from "./flapGuard.js"; import { withoutSnoozed } from "./alertSnoozes.js"; import { actionsFor, footerText, ntfyActionsHeader, type NotifyActions } from "./alertActionLinks.js"; import { firstMatchingRule, metaOfConditionKey, pushAccepts, type ChannelId, type NotifyMeta } from "./notifyRouting.js"; // ─── Channel senders (best-effort — failures are logged, never thrown) ────── async function sendGotify(title: string, message: string, actions?: NotifyActions | null) { const { gotify } = await getSettings(); if (!gotify.enabled || !gotify.url || !gotify.token) return; const base = gotify.url.replace(/\/$/, ""); try { const res = await fetch(`${base}/message?token=${encodeURIComponent(gotify.token)}`, { method: "POST", headers: { "Content-Type": "application/json" }, // Gotify has no buttons: a tap opens the page with them (and the address is in the text, for a client that does not tap). body: JSON.stringify({ title, message: actions ? message + footerText(actions) : message, priority: gotify.priority ?? 5, ...(actions ? { extras: { "client::notification": { click: { url: actions.pageUrl } } } } : {}) }), }); if (!res.ok) console.error(`[notify] Gotify error ${res.status}: ${await res.text().catch(() => "")}`); } catch (err) { console.error("[notify] Gotify failed:", err instanceof Error ? err.message : err); } } /** * A header can only carry Latin-1, and every title here has an em dash. ntfy reads RFC 2047 encoded words in its headers ("=?UTF-8?B?...?="), * so anything outside plain ASCII is sent that way. */ export function headerSafe(text: string): string { // eslint-disable-next-line no-control-regex return /^[ -~]*$/.test(text) ? text : `=?UTF-8?B?${Buffer.from(text, "utf8").toString("base64")}?=`; } async function sendNtfy(title: string, message: string, actions?: NotifyActions | null) { const { ntfy } = await getSettings(); if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return; const base = ntfy.url.replace(/\/$/, ""); const headers: Record = { "Content-Type": "text/plain", Title: headerSafe(title), Priority: String(ntfy.priority ?? 3), }; if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`; if (actions) { headers.Actions = ntfyActionsHeader(actions); headers.Click = actions.pageUrl; } try { const res = await fetch(`${base}/${encodeURIComponent(ntfy.topic)}`, { method: "POST", headers, body: message }); if (!res.ok) console.error(`[notify] ntfy error ${res.status}: ${await res.text().catch(() => "")}`); } catch (err) { console.error("[notify] ntfy failed:", err instanceof Error ? err.message : err); } } // ─── Telegram and Pushover ────────────────────────────────────────────────── // // Both are hosted services with one fixed address each (so nothing a person types decides where a request goes, as it does for a webhook). The addresses are kept here, in one object, so a test can // point them at a pretend server. Neither says why it refused in its HTTP status alone: Telegram answers { ok: false, description }, Pushover { status: 0, errors: [...] }, and that is what is shown. export const CHANNEL_ENDPOINTS = { telegram: "https://api.telegram.org", pushover: "https://api.pushover.net" }; const clip = (text: string, max: number) => (text.length <= max ? text : `${text.slice(0, max - 1)}…`); /** Telegram allows 4096 characters in a message. */ export const TELEGRAM_MAX = 4000; /** Pushover allows 1024 in a message, 250 in a title, 512 in a link. */ export const PUSHOVER_MESSAGE_MAX = 1024; export const PUSHOVER_TITLE_MAX = 250; export const PUSHOVER_URL_MAX = 512; const BUTTON_LABEL = "Acknowledge, silence or snooze"; /** Sends one message through a Telegram bot; throws what Telegram says when it refuses. The bot's token is in the address, so it is never put in an error. */ export async function telegramSend(cfg: { token: string; chatId: string; silent?: boolean }, text: string, actions?: NotifyActions | null): Promise { const post = async (body: Record) => { const res = await fetch(`${CHANNEL_ENDPOINTS.telegram}/bot${cfg.token}/sendMessage`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) }); const json = (await res.json().catch(() => null)) as { ok?: boolean; description?: string } | null; return { ok: res.ok && json?.ok !== false, status: res.status, description: json?.description ?? null }; }; const base = { chat_id: cfg.chatId, disable_web_page_preview: true, ...(cfg.silent ? { disable_notification: true } : {}) }; // A button that opens the page where the alert can be dealt with — plain text for what is said, so nothing in it needs escaping. let result = await post({ ...base, text: clip(text, TELEGRAM_MAX), ...(actions ? { reply_markup: { inline_keyboard: [[{ text: BUTTON_LABEL, url: actions.pageUrl }]] } } : {}) }); // Telegram refuses a button whose address it cannot open (an address on a private network, say): say it in the text instead. if (!result.ok && actions && result.status === 400) result = await post({ ...base, text: clip(text + footerText(actions), TELEGRAM_MAX) }); if (!result.ok) throw new Error(`Telegram: ${result.description ?? `HTTP ${result.status}`}`); } /** Sends one message through Pushover; throws what Pushover says when it refuses. */ export async function pushoverSend(cfg: { appToken: string; userKey: string; device?: string; priority?: number }, title: string, message: string, actions?: NotifyActions | null): Promise { const priority = Number.isInteger(cfg.priority) ? Math.min(2, Math.max(-2, cfg.priority as number)) : 0; const link = actions && actions.pageUrl.length <= PUSHOVER_URL_MAX ? actions.pageUrl : null; const body: Record = { token: cfg.appToken, user: cfg.userKey, title: clip(title, PUSHOVER_TITLE_MAX), // A link too long for Pushover goes in the text instead. message: clip(actions && !link ? message + footerText(actions) : message, PUSHOVER_MESSAGE_MAX), priority, }; if (cfg.device?.trim()) body.device = cfg.device.trim(); // An emergency message repeats until someone acknowledges it: Pushover wants to be told how often (every minute) and for how long (an hour). if (priority === 2) { body.retry = 60; body.expire = 3600; } if (link) { body.url = link; body.url_title = BUTTON_LABEL; } const res = await fetch(`${CHANNEL_ENDPOINTS.pushover}/1/messages.json`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) }); const json = (await res.json().catch(() => null)) as { status?: number; errors?: string[] } | null; if (!res.ok || json?.status !== 1) throw new Error(`Pushover: ${json?.errors?.length ? json.errors.join(", ") : `HTTP ${res.status}`}`); } async function sendTelegram(title: string, message: string, actions?: NotifyActions | null) { const { telegram } = await getSettings(); if (!telegram.enabled || !telegram.token || !telegram.chatId) return; try { await telegramSend(telegram, `${title}\n\n${message}`, actions); } catch (err) { console.error("[notify]", err instanceof Error ? err.message : "Telegram failed"); } } async function sendPushover(title: string, message: string, actions?: NotifyActions | null) { const { pushover } = await getSettings(); if (!pushover.enabled || !pushover.appToken || !pushover.userKey) return; try { await pushoverSend(pushover, title, message, actions); } catch (err) { console.error("[notify]", err instanceof Error ? err.message : "Pushover failed"); } } async function sendSmtp(title: string, message: string, actions?: NotifyActions | null) { const { smtp } = await getSettings(); if (!smtp.enabled || !smtp.host || !smtp.to || !smtp.from) return; try { const transporter = nodemailer.createTransport({ host: smtp.host, port: Number(smtp.port) || 587, secure: !!smtp.secure, auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined, }); await transporter.sendMail({ from: smtp.from, to: smtp.to, subject: title, text: actions ? message + footerText(actions) : message }); } catch (err) { console.error("[notify] SMTP failed:", err instanceof Error ? err.message : err); } } async function sendWebhook(title: string, message: string, actions?: NotifyActions | null) { const { webhook } = await getSettings(); if (!webhook.enabled || !webhook.url) return; const headers: Record = { "Content-Type": "application/json" }; if (webhook.secret) headers["X-Webhook-Secret"] = webhook.secret; try { const res = await fetch(webhook.url, { method: "POST", headers, body: JSON.stringify({ content: `**${title}**\n${message}${actions ? footerText(actions) : ""}` }), }); if (!res.ok) console.error(`[notify] Webhook error ${res.status}: ${await res.text().catch(() => "")}`); } catch (err) { console.error("[notify] Webhook failed:", err instanceof Error ? err.message : err); } } async function sendWebPush(title: string, message: string, only: number[] | null, actions?: NotifyActions | null) { const { webPush } = await getSettings(); if (!webPush.enabled) return; // For one alert, buttons on the notification itself (browsers show two); for several, a tap opens the app. const buttons = actions?.direct.slice(0, 2).map((d) => ({ action: d.id, title: d.label, path: d.path })); const message_ = { title, body: message, ...(buttons?.length ? { actions: buttons } : {}) }; try { if (only === null) await sendPush(message_); else if (only.length > 0) await sendPush(message_, await db.select().from(pushSubscriptions).where(inArray(pushSubscriptions.id, only))); } catch (err) { console.error("[notify] Web push failed:", err instanceof Error ? err.message : err); } } /** Email to one address, through the email channel's server, whatever the channel's own recipient is (an escalation to someone else). */ async function sendExtraEmail(title: string, message: string, to: string) { const { smtp } = await getSettings(); if (!smtp.host || !smtp.from) { console.error("[notify] An escalation wants to email someone, but email isn't set up."); return; } try { const transporter = nodemailer.createTransport({ host: smtp.host, port: Number(smtp.port) || 587, secure: !!smtp.secure, auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined, }); await transporter.sendMail({ from: smtp.from, to, subject: title, text: message }); } catch (err) { console.error("[notify] Extra email failed:", err instanceof Error ? err.message : err); } } // ─── Who a notification is for ───────────────────────────────────────────── /** The channels and browsers a notification goes to. `null` means every one that is on (the way it was before routing existed). */ export interface Audience { channels: ChannelId[] | null; /** The subscribed browsers that want it; null = all of them. */ pushIds: number[] | null; /** Also email this address. */ email?: string; } export const EVERYONE: Audience = { channels: null, pushIds: null }; export const audienceKey = (a: Audience) => JSON.stringify({ c: a.channels ? [...a.channels].sort() : null, p: a.pushIds ? [...a.pushIds].sort((x, y) => x - y) : null, e: a.email ?? null }); /** Told to nobody: a rule said so (the Alerts page still shows it). */ const isMuted = (a: Audience) => a.channels !== null && a.channels.length === 0 && !a.email; /** Works out who a notification is for: the first routing rule it matches picks the channels, and browsers may turn down what they don't want. */ export async function audienceFor(meta?: NotifyMeta): Promise { if (!meta) return EVERYONE; const { routing } = await getSettings(); let channels: ChannelId[] | null = null; if (routing.rules.some((r) => r.enabled)) { let tags = meta.tags; if (!tags && meta.serverId != null) { const [row] = await db.select({ tags: servers.tags }).from(servers).where(inArray(servers.id, [meta.serverId])); tags = parseTags(row?.tags); } channels = firstMatchingRule(routing.rules, { ...meta, tags: tags ?? [] })?.channels ?? null; } let pushIds: number[] | null = null; if (channels === null || channels.includes("webPush")) { const subs = await db.select({ id: pushSubscriptions.id, minSeverity: pushSubscriptions.minSeverity, categories: pushSubscriptions.categories }).from(pushSubscriptions); // Only when somebody has said what they want; otherwise everyone gets it, as ever. if (subs.some((s) => s.minSeverity || s.categories)) pushIds = subs.filter((s) => pushAccepts(s, meta)).map((s) => s.id); } return { channels, pushIds }; } async function sendToChannels(title: string, message: string, audience: Audience = EVERYONE, actions: NotifyActions | null = null): Promise { const to = (c: ChannelId) => audience.channels === null || audience.channels.includes(c); await Promise.all([ to("gotify") ? sendGotify(title, message, actions) : undefined, to("ntfy") ? sendNtfy(title, message, actions) : undefined, to("telegram") ? sendTelegram(title, message, actions) : undefined, to("pushover") ? sendPushover(title, message, actions) : undefined, to("smtp") ? sendSmtp(title, message, actions) : undefined, to("webhook") ? sendWebhook(title, message, actions) : undefined, to("webPush") ? sendWebPush(title, message, audience.pushIds, actions) : undefined, audience.email ? sendExtraEmail(title, actions ? message + footerText(actions) : message, audience.email) : undefined, ]); } /** * Sends to an audience that is already worked out: held for the quiet-hours digest if it is quiet (unless told not to), nowhere if a rule * said nobody. Escalations use this with their own channels. */ export async function notifyAudience(title: string, message: string, audience: Audience, options: { quietHours?: boolean; actions?: NotifyActions | null; critical?: boolean } = {}): Promise { if (isMuted(audience)) return; // Held when quiet hours or an away stretch say so — unless it is critical and they let critical through (services/availability.ts). if (options.quietHours !== false && (await holdReason(options.critical === true))) { await enqueueNotification(title, message, audience.channels === null && audience.pushIds === null && !audience.email ? null : JSON.stringify(audience)); return; } await sendToChannels(title, message, audience, options.actions ?? null); } /** Items that may each be for a different audience (a rule can send a disk warning one place and a server outage another): sent as one message per audience. */ async function notifyGrouped(items: T[], metaOf: (item: T) => NotifyMeta, build: (group: T[]) => { title: string; message: string }, keysOf?: (item: T) => string | undefined): Promise { const groups = new Map(); for (const item of items) { const audience = await audienceFor(metaOf(item)); const key = audienceKey(audience); const group = groups.get(key) ?? { audience, items: [] }; group.items.push(item); groups.set(key, group); } for (const group of groups.values()) { const { title, message } = build(group.items); // Buttons, when the message is about alerts that have a key and an administrator has switched them on (a message held for quiet hours is sent without them). const critical = group.items.some((i) => metaOf(i).severity === "critical"); await notifyAudience(title, message, group.audience, { critical, ...(keysOf ? { actions: await actionsFor(group.items.map(keysOf)) } : {}) }); } } /** Sends at once through every channel that is switched on, whatever the time — for something a person just asked for. */ export async function notifyNow(title: string, message: string): Promise { await sendToChannels(title, message); } /** How many channels are switched on and filled in enough to send, so a "send it now" can say when there is nowhere to send to. */ export async function enabledChannelCount(): Promise { const { gotify, ntfy, telegram, pushover, smtp, webhook, webPush } = await getSettings(); return [ webPush.enabled && (await subscriptionCount()) > 0, gotify.enabled && gotify.url && gotify.token, ntfy.enabled && ntfy.url && ntfy.topic, telegram.enabled && telegram.token && telegram.chatId, pushover.enabled && pushover.appToken && pushover.userKey, smtp.enabled && smtp.host && smtp.to && smtp.from, webhook.enabled && webhook.url, ].filter(Boolean).length; } /** `meta` says what it is about, so routing rules and browsers' wishes can apply; without it, it goes to everyone. */ export async function notify(title: string, message: string, meta?: NotifyMeta): Promise { await notifyAudience(title, message, await audienceFor(meta), { critical: meta?.severity === "critical" }); } /** * Sends everything held as one digest per audience, then clears the queue: called when quiet hours end, when an away stretch ends, or by hand (`force`). What is held for one * reason is not let out while the other still holds it: quiet hours ending in the middle of a holiday release nothing, and coming back at 02:00 releases nothing until morning. */ export async function flushQuietHoursQueue(options: { reason?: "quiet" | "away"; force?: boolean } = {}): Promise { const reason = options.reason ?? "quiet"; if (!options.force) { const s = await getSettings(); const held = holdFor({ away: s.away, quietHours: s.quietHours, timezone: s.notifications.timezone }, false, new Date()); if (held !== null) return 0; } const queued = await listQueuedNotifications(); if (queued.length === 0) return 0; // One digest for each audience the held notifications were for. const groups = new Map(); for (const q of queued) groups.set(q.audience ?? "", [...(groups.get(q.audience ?? "") ?? []), q]); for (const [stored, items] of groups) { let audience: Audience = EVERYONE; try { if (stored) audience = JSON.parse(stored) as Audience; } catch { // unreadable: everyone, as before } const sections = items.map((q) => `— ${q.title} —\n${q.message}`); await sendToChannels( reason === "away" ? "Homelab Manager — Notifications from while you were away" : "Homelab Manager — Notifications from quiet hours", `${items.length} notification${items.length !== 1 ? "s were" : " was"} held ${reason === "away" ? "while you were away" : "during quiet hours"}:\n\n${sections.join("\n\n")}`, audience, ); } await clearQueuedNotifications(); return queued.length; } // ─── Test senders — throw on failure so the route can report it ──────────── export async function testGotify(cfg: { url: string; token: string; priority?: number }): Promise { const base = cfg.url.replace(/\/$/, ""); const res = await fetch(`${base}/message?token=${encodeURIComponent(cfg.token)}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ title: "Homelab Manager — Test", message: "Gotify notifications are working correctly.", priority: cfg.priority ?? 5, }), }); if (!res.ok) throw new Error(`Gotify returned ${res.status}: ${await res.text().catch(() => "")}`); } export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise { const base = cfg.url.replace(/\/$/, ""); const headers: Record = { "Content-Type": "text/plain", Title: headerSafe("Homelab Manager — Test"), Priority: String(cfg.priority ?? 3) }; if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`; const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, { method: "POST", headers, body: "ntfy notifications are working correctly.", }); if (!res.ok) throw new Error(`ntfy returned ${res.status}: ${await res.text().catch(() => "")}`); } export async function testTelegram(cfg: { token: string; chatId: string; silent?: boolean }): Promise { await telegramSend(cfg, "Homelab Manager — Test\n\nTelegram notifications are working correctly."); } export async function testPushover(cfg: { appToken: string; userKey: string; device?: string; priority?: number }): Promise { await pushoverSend(cfg, "Homelab Manager — Test", "Pushover notifications are working correctly."); } export async function testSmtp(cfg: { host: string; port?: number; secure?: boolean; username?: string; password?: string; from: string; to: string; }): Promise { const transporter = nodemailer.createTransport({ host: cfg.host, port: Number(cfg.port) || 587, secure: !!cfg.secure, auth: cfg.username ? { user: cfg.username, pass: cfg.password } : undefined, }); await transporter.verify(); await transporter.sendMail({ from: cfg.from, to: cfg.to, subject: "Homelab Manager — Test", text: "SMTP notifications are working correctly." }); } export async function testWebhook(cfg: { url: string; secret?: string }): Promise { const headers: Record = { "Content-Type": "application/json" }; if (cfg.secret) headers["X-Webhook-Secret"] = cfg.secret; const res = await fetch(cfg.url, { method: "POST", headers, body: JSON.stringify({ content: "**Homelab Manager — Test**\nWebhook notifications are working correctly." }), }); if (!res.ok) throw new Error(`Webhook returned ${res.status}: ${await res.text().catch(() => "")}`); } // ─── Event helpers ─────────────────────────────────────────────────────────── async function eventEnabled(key: keyof NotificationEvents): Promise { const { notifications } = await getSettings(); return notifications[key] !== false; } export async function notifyDnsRecordAdded( providerLabel: string, zoneName: string, record: { type: string; name: string; content: string }, ): Promise { if (!(await eventEnabled("dnsAdd"))) return; await notify("DNS Record Added", `[${providerLabel}] ${zoneName}\n+ ${record.type} ${record.name} → ${record.content}`); } export async function notifyDnsRecordUpdated( providerLabel: string, zoneName: string, record: { type: string; name: string; content: string }, ): Promise { if (!(await eventEnabled("dnsUpdate"))) return; await notify("DNS Record Updated", `[${providerLabel}] ${zoneName}\n✎ ${record.type} ${record.name} → ${record.content}`); } export async function notifyDnsRecordDeleted( providerLabel: string, zoneName: string, record: { type: string; name: string; content: string }, ): Promise { if (!(await eventEnabled("dnsDelete"))) return; await notify("DNS Record Deleted", `[${providerLabel}] ${zoneName}\n− ${record.type} ${record.name} ${record.content}`); } export async function notifySecretExpiry( expiring: { name: string; status: "expired" | "expiring"; daysLeft: number }[], checkFailures: { name: string; host: string; port: number; error?: string }[] = [], ): Promise { if (expiring.length === 0 && checkFailures.length === 0) return; if (!(await eventEnabled("secretCheck"))) return; const sections: string[] = []; if (expiring.length > 0) { const lines = expiring.map((s) => (s.status === "expired" ? `✕ EXPIRED — ${s.name}` : `⚠ ${s.daysLeft}d left — ${s.name}`)); sections.push(`${expiring.length} secret${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`); } if (checkFailures.length > 0) { const lines = checkFailures.map((f) => `⚠ ${f.name} (${f.host}:${f.port}) — ${f.error ?? "check failed"}`); sections.push( `Couldn't read the live certificate for ${checkFailures.length} monitored secret${checkFailures.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`, ); } await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"), { category: "expiry", severity: expiring.some((s) => s.status === "expired") ? "critical" : "warning" }); } export async function notifyDockerUpdates( updatesAvailable: { integrationName: string; containerName: string; environmentName: string; newerVersion: string | null }[], ): Promise { if (updatesAvailable.length === 0) return; if (!(await eventEnabled("dockerUpdateCheck"))) return; const lines = updatesAvailable.map( (u) => `${u.containerName} [${u.environmentName}, ${u.integrationName}]${u.newerVersion ? ` → ${u.newerVersion}` : ""}`, ); await notify( "Homelab Manager — Docker Updates Available", `${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`, { category: "updates", severity: "warning" }, ); } export async function notifyProxmoxBackupFailure( failures: { integrationName: string; node: string; guestId: string | null; status: string }[], ): Promise { if (failures.length === 0) return; if (!(await eventEnabled("proxmoxBackupCheck"))) return; const lines = failures.map( (f) => `${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} [${f.integrationName}]: ${f.status}`, ); await notify( "Homelab Manager — Proxmox Backup Failed", `${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`, { category: "backup", severity: "warning" }, ); } export async function notifyProxmoxUncoveredGuests( uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[], ): Promise { if (uncovered.length === 0) return; if (!(await eventEnabled("proxmoxBackupCheck"))) return; const lines = uncovered.map((u) => `${u.guestName} (#${u.vmid}) on ${u.node} [${u.integrationName}]`); await notify( "Homelab Manager — Proxmox Guests Without Backup Coverage", `${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`, { category: "backup", severity: "warning" }, ); } export async function notifyStaleBackups(stale: { message: string }[]): Promise { if (stale.length === 0) return; if (!(await eventEnabled("proxmoxBackupCheck"))) return; await notify( "Homelab Manager — Backups Gone Stale", `${stale.length} guest${stale.length !== 1 ? "s have" : " has"} no recent backup:\n\n${stale.map((s) => s.message).join("\n")}`, { category: "backup", severity: "warning" }, ); } export async function notifyUpdateRunFinished(run: { serverName: string; templateName: string; ok: boolean; message: string }): Promise { if (!(await eventEnabled("updateRunResults"))) return; await notify( run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`, `"${run.templateName}" on ${run.serverName}: ${run.message}`, { category: "updates", severity: run.ok ? "warning" : "critical" }, ); } /** How a patch plan's run ended: one message, with a line for each server. Held back or sent with the other update-run results. */ export async function notifyPatchPlanFinished(run: { planName: string; ok: boolean; message: string; lines: string[] }): Promise { if (!(await eventEnabled("updateRunResults"))) return; await notify( run.ok ? `Homelab Manager — Patch plan "${run.planName}" finished` : `Homelab Manager — Patch plan "${run.planName}" didn't finish cleanly`, `${run.message}${run.lines.length > 0 ? `\n\n${run.lines.join("\n")}` : ""}`, { category: "updates", severity: run.ok ? "warning" : "critical" }, ); } export async function notifyPbsVerificationFailed( failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[], ): Promise { if (failures.length === 0) return; if (!(await eventEnabled("pbsVerificationCheck"))) return; const lines = failures.map((f) => f.error ? `${f.datastore} [${f.integrationName}]: couldn't be read — ${f.error}` : `${f.datastore} [${f.integrationName}]: ${f.failedCount} snapshot${f.failedCount !== 1 ? "s" : ""} failed verification`, ); await notify( "Homelab Manager — Proxmox Backup Server Verification Failed", `${failures.length} datastore${failures.length !== 1 ? "s have" : " has"} a problem:\n\n${lines.join("\n")}`, { category: "backup", severity: "warning" }, ); } /** What a problem is about, for routing: its key says the kind and the server; the rest follows its severity. */ const issueMeta = (i: { key?: string; severity?: string }): NotifyMeta => (i.key ? metaOfConditionKey(i.key, i.severity) : { category: null, severity: i.severity === "critical" ? "critical" : "warning" }); export async function notifyHealthIssues(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise { // The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told. for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } }); if (issues.length === 0) return; // One somebody has snoozed is not announced now; if it is still there when the snooze ends, it is announced then (services/alertSnoozes.ts). issues = await withoutSnoozed(issues); if (issues.length === 0) return; // An alert that keeps coming and going is announced once as "flapping" and then left quiet (services/flapGuard.ts); that is tracked whether or not this is switched on. const { pass, started } = await dampRaised(issues); if (!(await eventEnabled("healthAlerts"))) return; if (started.length > 0) await notifyFlapping(started); if (pass.length === 0) return; issues = pass; await notifyGrouped(issues, issueMeta, (group) => ({ title: "Homelab Manager — Health Alert", message: `${group.length} new issue${group.length !== 1 ? "s" : ""}:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`, }), (i) => i.key); } export async function notifyHealthRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise { for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } }); if (resolved.length === 0) return; resolved = await withoutSnoozed(resolved); if (resolved.length === 0) return; const passed = await dampRecovered(resolved); if (!(await eventEnabled("healthAlerts"))) return; if (passed.length === 0) return; resolved = passed; await notifyGrouped(resolved, issueMeta, (group) => ({ title: "Homelab Manager — Health Recovered", message: `${group.length} issue${group.length !== 1 ? "s" : ""} cleared:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`, })); } /** Alerts that were snoozed, whose time is up and that are still there: said once, as a reminder. */ export async function notifySnoozeEnded(items: { message: string; key?: string; severity?: string }[]): Promise { if (items.length === 0) return; await notifyGrouped(items, issueMeta, (group) => ({ title: "Homelab Manager — Snooze ended", message: `${group.length === 1 ? "An alert you snoozed is" : `${group.length} alerts you snoozed are`} still there:\n\n${group.map((i) => `⏰ ${i.message}`).join("\n")}`, }), (i) => i.key); } /** Alerts that somebody closed and that have since got more serious: opened again, and said once. */ export async function notifyClosedWorse(items: { message: string; key?: string; severity?: string }[]): Promise { if (items.length === 0) return; await notifyGrouped(items, issueMeta, (group) => ({ title: "Homelab Manager — Closed alert got worse", message: `${group.length === 1 ? "An alert you closed is" : `${group.length} alerts you closed are`} more serious now, so ${group.length === 1 ? "it is" : "they are"} open again:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`, }), (i) => i.key); } /** An alert that has just been called flapping: said once, with how often it has come, instead of each time. */ export async function notifyFlapping(items: { message: string; key?: string; severity?: string; count: number }[]): Promise { if (items.length === 0) return; await notifyGrouped(items, issueMeta, (group) => ({ title: "Homelab Manager — Flapping alert", message: `${group.length === 1 ? "An alert keeps" : `${group.length} alerts keep`} coming and going:\n\n${group.map((i) => `↻ ${i.message} — raised ${i.count} times within an hour`).join("\n")}\n\nIts further messages are held back until it has been quiet for an hour; it stays on the Alerts page, marked as flapping.`, }), (i) => i.key); } /** Flapping alerts that have been quiet for an hour: what state each is in now. */ export async function notifySettled(items: { message: string; key: string; active: boolean }[]): Promise { if (items.length === 0 || !(await eventEnabled("healthAlerts"))) return; await notifyGrouped(items, issueMeta, (group) => ({ title: "Homelab Manager — Alert settled", message: `${group.length === 1 ? "An alert that was flapping has" : `${group.length} alerts that were flapping have`} been steady for an hour:\n\n${group.map((i) => `${i.active ? "⚠ still active" : "✓ cleared"} — ${i.message}`).join("\n")}`, })); } export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise { // The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told. for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } }); if (issues.length === 0) return; issues = await withoutSnoozed(issues); if (issues.length === 0 || !(await eventEnabled("automationAlerts"))) return; await notifyGrouped(issues, issueMeta, (group) => ({ title: "Homelab Manager — Automation Failed", message: `${group.length} run${group.length !== 1 ? "s" : ""} failed:\n\n${group.map((i) => `✕ ${i.message}`).join("\n")}`, }), (i) => i.key); } export async function notifyAutomationRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise { for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } }); if (resolved.length === 0) return; resolved = await withoutSnoozed(resolved); if (resolved.length === 0 || !(await eventEnabled("automationAlerts"))) return; await notifyGrouped(resolved, issueMeta, (group) => ({ title: "Homelab Manager — Automation Recovered", message: `${group.length} no longer failing:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`, })); } export async function notifyDomainExpiry( expiring: { name: string; status: "expired" | "expiring"; daysLeft: number; expiresAt: string; registrar: string | null }[], staleChecks: { name: string; error: string; lastOkAt: string | null }[] = [], ): Promise { if (expiring.length === 0 && staleChecks.length === 0) return; if (!(await eventEnabled("domainExpiryCheck"))) return; const sections: string[] = []; if (expiring.length > 0) { const lines = expiring.map((d) => { const registrar = d.registrar ? ` [${d.registrar}]` : ""; return d.status === "expired" ? `✕ EXPIRED ${d.expiresAt} — ${d.name}${registrar}` : `⚠ ${d.daysLeft}d left (${d.expiresAt}) — ${d.name}${registrar}`; }); sections.push(`${expiring.length} domain${expiring.length !== 1 ? "s" : ""} need renewing:\n\n${lines.join("\n")}`); } if (staleChecks.length > 0) { const lines = staleChecks.map((s) => `⚠ ${s.name} — ${s.error}${s.lastOkAt ? ` (last read ${s.lastOkAt.slice(0, 10)})` : " (never read)"}`); sections.push( `Couldn't refresh the registration for ${staleChecks.length} domain${staleChecks.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`, ); } await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"), { category: "expiry", severity: expiring.some((d) => d.status === "expired") ? "critical" : "warning" }); } export async function notifyTailscaleKeyExpiry( expiring: { integrationName: string; deviceLabel: string; daysLeft: number }[], ): Promise { if (expiring.length === 0) return; if (!(await eventEnabled("tailscaleKeyCheck"))) return; const lines = expiring.map( (d) => `${d.daysLeft < 0 ? "✕ EXPIRED" : `⚠ ${d.daysLeft}d left`} — ${d.deviceLabel} [${d.integrationName}]`, ); await notify( "Homelab Manager — Tailscale Key Expiry", `${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`, { category: "expiry", severity: expiring.some((d) => d.daysLeft < 0) ? "critical" : "warning" }, ); } // Diagnostic-log "source" values (see integrations/*/adapter.ts and dns/adapters/*.ts) to display names. const SOURCE_LABELS: Record = { cloudflare: "Cloudflare", loopia: "Loopia", pihole: "Pi-hole", azure: "Azure DNS", cpanel: "cPanel", technitium: "Technitium", tailscale: "Tailscale", proxmox: "Proxmox", synology: "Synology", semaphore: "Semaphore", gitea: "Gitea", dockhand: "Dockhand", uptimekuma: "Uptime Kuma", phpipam: "phpIPAM", pbs: "Proxmox Backup Server", osticket: "osTicket", unifi: "UniFi", npm: "Nginx Proxy Manager", homeassistant: "Home Assistant", }; export function sourceLabel(source: string): string { return SOURCE_LABELS[source] ?? source; } export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise { emitEvent("alert.opened", { alert: { key: `integration:${source}`, source, severity: "warning", message: `${sourceLabel(source)} has failed its last ${consecutiveFailures} calls in a row.` } }); if (!(await eventEnabled("integrationFailureAlerts"))) return; await notify( "Homelab Manager — Integration Down", `${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`, { category: "integration", severity: "warning" }, ); } export async function notifyIntegrationRecovered(source: string): Promise { emitEvent("alert.resolved", { alert: { key: `integration:${source}`, message: `${sourceLabel(source)} is answering again.` } }); if (!(await eventEnabled("integrationFailureAlerts"))) return; await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`, { category: "integration", severity: "warning" }); }