Both send to one fixed address each (api.telegram.org, api.pushover.net), take part in routing rules and the channel count, have a Send test that shows the service's own refusal, and keep their tokens and keys encrypted and out of errors and the audit log. A Telegram message carries one button, and a Pushover message one link, to the page where an alert can be acknowledged, silenced or snoozed. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
743 lines
39 KiB
TypeScript
743 lines
39 KiB
TypeScript
import nodemailer from "nodemailer";
|
||
import { getSettings, type NotificationEvents } from "./settingsStore.js";
|
||
import { holdFor, holdReason } from "./availability.js";
|
||
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
|
||
import { emitEvent } from "./eventWebhooks.js";
|
||
import { sendPush, subscriptionCount } from "./webPush.js";
|
||
import { db } from "../db/client.js";
|
||
import { inArray } from "drizzle-orm";
|
||
import { pushSubscriptions, servers } from "../db/schema.js";
|
||
import { parseTags } from "./serverTags.js";
|
||
import { dampRaised, dampRecovered } from "./flapGuard.js";
|
||
import { withoutSnoozed } from "./alertSnoozes.js";
|
||
import { actionsFor, footerText, ntfyActionsHeader, type NotifyActions } from "./alertActionLinks.js";
|
||
import { firstMatchingRule, metaOfConditionKey, pushAccepts, type ChannelId, type NotifyMeta } from "./notifyRouting.js";
|
||
|
||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||
|
||
async function sendGotify(title: string, message: string, actions?: NotifyActions | null) {
|
||
const { gotify } = await getSettings();
|
||
if (!gotify.enabled || !gotify.url || !gotify.token) return;
|
||
const base = gotify.url.replace(/\/$/, "");
|
||
try {
|
||
const res = await fetch(`${base}/message?token=${encodeURIComponent(gotify.token)}`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
// Gotify has no buttons: a tap opens the page with them (and the address is in the text, for a client that does not tap).
|
||
body: JSON.stringify({ title, message: actions ? message + footerText(actions) : message, priority: gotify.priority ?? 5, ...(actions ? { extras: { "client::notification": { click: { url: actions.pageUrl } } } } : {}) }),
|
||
});
|
||
if (!res.ok) console.error(`[notify] Gotify error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] Gotify failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* A header can only carry Latin-1, and every title here has an em dash. ntfy reads RFC 2047 encoded words in its headers ("=?UTF-8?B?...?="),
|
||
* so anything outside plain ASCII is sent that way.
|
||
*/
|
||
export function headerSafe(text: string): string {
|
||
// eslint-disable-next-line no-control-regex
|
||
return /^[ -~]*$/.test(text) ? text : `=?UTF-8?B?${Buffer.from(text, "utf8").toString("base64")}?=`;
|
||
}
|
||
|
||
async function sendNtfy(title: string, message: string, actions?: NotifyActions | null) {
|
||
const { ntfy } = await getSettings();
|
||
if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return;
|
||
const base = ntfy.url.replace(/\/$/, "");
|
||
const headers: Record<string, string> = {
|
||
"Content-Type": "text/plain",
|
||
Title: headerSafe(title),
|
||
Priority: String(ntfy.priority ?? 3),
|
||
};
|
||
if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`;
|
||
if (actions) {
|
||
headers.Actions = ntfyActionsHeader(actions);
|
||
headers.Click = actions.pageUrl;
|
||
}
|
||
try {
|
||
const res = await fetch(`${base}/${encodeURIComponent(ntfy.topic)}`, { method: "POST", headers, body: message });
|
||
if (!res.ok) console.error(`[notify] ntfy error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] ntfy failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
// ─── Telegram and Pushover ──────────────────────────────────────────────────
|
||
//
|
||
// Both are hosted services with one fixed address each (so nothing a person types decides where a request goes, as it does for a webhook). The addresses are kept here, in one object, so a test can
|
||
// point them at a pretend server. Neither says why it refused in its HTTP status alone: Telegram answers { ok: false, description }, Pushover { status: 0, errors: [...] }, and that is what is shown.
|
||
|
||
export const CHANNEL_ENDPOINTS = { telegram: "https://api.telegram.org", pushover: "https://api.pushover.net" };
|
||
|
||
const clip = (text: string, max: number) => (text.length <= max ? text : `${text.slice(0, max - 1)}…`);
|
||
|
||
/** Telegram allows 4096 characters in a message. */
|
||
export const TELEGRAM_MAX = 4000;
|
||
/** Pushover allows 1024 in a message, 250 in a title, 512 in a link. */
|
||
export const PUSHOVER_MESSAGE_MAX = 1024;
|
||
export const PUSHOVER_TITLE_MAX = 250;
|
||
export const PUSHOVER_URL_MAX = 512;
|
||
const BUTTON_LABEL = "Acknowledge, silence or snooze";
|
||
|
||
/** Sends one message through a Telegram bot; throws what Telegram says when it refuses. The bot's token is in the address, so it is never put in an error. */
|
||
export async function telegramSend(cfg: { token: string; chatId: string; silent?: boolean }, text: string, actions?: NotifyActions | null): Promise<void> {
|
||
const post = async (body: Record<string, unknown>) => {
|
||
const res = await fetch(`${CHANNEL_ENDPOINTS.telegram}/bot${cfg.token}/sendMessage`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) });
|
||
const json = (await res.json().catch(() => null)) as { ok?: boolean; description?: string } | null;
|
||
return { ok: res.ok && json?.ok !== false, status: res.status, description: json?.description ?? null };
|
||
};
|
||
const base = { chat_id: cfg.chatId, disable_web_page_preview: true, ...(cfg.silent ? { disable_notification: true } : {}) };
|
||
// A button that opens the page where the alert can be dealt with — plain text for what is said, so nothing in it needs escaping.
|
||
let result = await post({ ...base, text: clip(text, TELEGRAM_MAX), ...(actions ? { reply_markup: { inline_keyboard: [[{ text: BUTTON_LABEL, url: actions.pageUrl }]] } } : {}) });
|
||
// Telegram refuses a button whose address it cannot open (an address on a private network, say): say it in the text instead.
|
||
if (!result.ok && actions && result.status === 400) result = await post({ ...base, text: clip(text + footerText(actions), TELEGRAM_MAX) });
|
||
if (!result.ok) throw new Error(`Telegram: ${result.description ?? `HTTP ${result.status}`}`);
|
||
}
|
||
|
||
/** Sends one message through Pushover; throws what Pushover says when it refuses. */
|
||
export async function pushoverSend(cfg: { appToken: string; userKey: string; device?: string; priority?: number }, title: string, message: string, actions?: NotifyActions | null): Promise<void> {
|
||
const priority = Number.isInteger(cfg.priority) ? Math.min(2, Math.max(-2, cfg.priority as number)) : 0;
|
||
const link = actions && actions.pageUrl.length <= PUSHOVER_URL_MAX ? actions.pageUrl : null;
|
||
const body: Record<string, string | number> = {
|
||
token: cfg.appToken,
|
||
user: cfg.userKey,
|
||
title: clip(title, PUSHOVER_TITLE_MAX),
|
||
// A link too long for Pushover goes in the text instead.
|
||
message: clip(actions && !link ? message + footerText(actions) : message, PUSHOVER_MESSAGE_MAX),
|
||
priority,
|
||
};
|
||
if (cfg.device?.trim()) body.device = cfg.device.trim();
|
||
// An emergency message repeats until someone acknowledges it: Pushover wants to be told how often (every minute) and for how long (an hour).
|
||
if (priority === 2) {
|
||
body.retry = 60;
|
||
body.expire = 3600;
|
||
}
|
||
if (link) {
|
||
body.url = link;
|
||
body.url_title = BUTTON_LABEL;
|
||
}
|
||
const res = await fetch(`${CHANNEL_ENDPOINTS.pushover}/1/messages.json`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) });
|
||
const json = (await res.json().catch(() => null)) as { status?: number; errors?: string[] } | null;
|
||
if (!res.ok || json?.status !== 1) throw new Error(`Pushover: ${json?.errors?.length ? json.errors.join(", ") : `HTTP ${res.status}`}`);
|
||
}
|
||
|
||
async function sendTelegram(title: string, message: string, actions?: NotifyActions | null) {
|
||
const { telegram } = await getSettings();
|
||
if (!telegram.enabled || !telegram.token || !telegram.chatId) return;
|
||
try {
|
||
await telegramSend(telegram, `${title}\n\n${message}`, actions);
|
||
} catch (err) {
|
||
console.error("[notify]", err instanceof Error ? err.message : "Telegram failed");
|
||
}
|
||
}
|
||
|
||
async function sendPushover(title: string, message: string, actions?: NotifyActions | null) {
|
||
const { pushover } = await getSettings();
|
||
if (!pushover.enabled || !pushover.appToken || !pushover.userKey) return;
|
||
try {
|
||
await pushoverSend(pushover, title, message, actions);
|
||
} catch (err) {
|
||
console.error("[notify]", err instanceof Error ? err.message : "Pushover failed");
|
||
}
|
||
}
|
||
|
||
async function sendSmtp(title: string, message: string, actions?: NotifyActions | null) {
|
||
const { smtp } = await getSettings();
|
||
if (!smtp.enabled || !smtp.host || !smtp.to || !smtp.from) return;
|
||
try {
|
||
const transporter = nodemailer.createTransport({
|
||
host: smtp.host,
|
||
port: Number(smtp.port) || 587,
|
||
secure: !!smtp.secure,
|
||
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
|
||
});
|
||
await transporter.sendMail({ from: smtp.from, to: smtp.to, subject: title, text: actions ? message + footerText(actions) : message });
|
||
} catch (err) {
|
||
console.error("[notify] SMTP failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendWebhook(title: string, message: string, actions?: NotifyActions | null) {
|
||
const { webhook } = await getSettings();
|
||
if (!webhook.enabled || !webhook.url) return;
|
||
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
||
if (webhook.secret) headers["X-Webhook-Secret"] = webhook.secret;
|
||
try {
|
||
const res = await fetch(webhook.url, {
|
||
method: "POST",
|
||
headers,
|
||
body: JSON.stringify({ content: `**${title}**\n${message}${actions ? footerText(actions) : ""}` }),
|
||
});
|
||
if (!res.ok) console.error(`[notify] Webhook error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] Webhook failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendWebPush(title: string, message: string, only: number[] | null, actions?: NotifyActions | null) {
|
||
const { webPush } = await getSettings();
|
||
if (!webPush.enabled) return;
|
||
// For one alert, buttons on the notification itself (browsers show two); for several, a tap opens the app.
|
||
const buttons = actions?.direct.slice(0, 2).map((d) => ({ action: d.id, title: d.label, path: d.path }));
|
||
const message_ = { title, body: message, ...(buttons?.length ? { actions: buttons } : {}) };
|
||
try {
|
||
if (only === null) await sendPush(message_);
|
||
else if (only.length > 0) await sendPush(message_, await db.select().from(pushSubscriptions).where(inArray(pushSubscriptions.id, only)));
|
||
} catch (err) {
|
||
console.error("[notify] Web push failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
/** Email to one address, through the email channel's server, whatever the channel's own recipient is (an escalation to someone else). */
|
||
async function sendExtraEmail(title: string, message: string, to: string) {
|
||
const { smtp } = await getSettings();
|
||
if (!smtp.host || !smtp.from) {
|
||
console.error("[notify] An escalation wants to email someone, but email isn't set up.");
|
||
return;
|
||
}
|
||
try {
|
||
const transporter = nodemailer.createTransport({
|
||
host: smtp.host,
|
||
port: Number(smtp.port) || 587,
|
||
secure: !!smtp.secure,
|
||
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
|
||
});
|
||
await transporter.sendMail({ from: smtp.from, to, subject: title, text: message });
|
||
} catch (err) {
|
||
console.error("[notify] Extra email failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
// ─── Who a notification is for ─────────────────────────────────────────────
|
||
|
||
/** The channels and browsers a notification goes to. `null` means every one that is on (the way it was before routing existed). */
|
||
export interface Audience {
|
||
channels: ChannelId[] | null;
|
||
/** The subscribed browsers that want it; null = all of them. */
|
||
pushIds: number[] | null;
|
||
/** Also email this address. */
|
||
email?: string;
|
||
}
|
||
|
||
export const EVERYONE: Audience = { channels: null, pushIds: null };
|
||
|
||
export const audienceKey = (a: Audience) => JSON.stringify({ c: a.channels ? [...a.channels].sort() : null, p: a.pushIds ? [...a.pushIds].sort((x, y) => x - y) : null, e: a.email ?? null });
|
||
|
||
/** Told to nobody: a rule said so (the Alerts page still shows it). */
|
||
const isMuted = (a: Audience) => a.channels !== null && a.channels.length === 0 && !a.email;
|
||
|
||
/** Works out who a notification is for: the first routing rule it matches picks the channels, and browsers may turn down what they don't want. */
|
||
export async function audienceFor(meta?: NotifyMeta): Promise<Audience> {
|
||
if (!meta) return EVERYONE;
|
||
const { routing } = await getSettings();
|
||
let channels: ChannelId[] | null = null;
|
||
if (routing.rules.some((r) => r.enabled)) {
|
||
let tags = meta.tags;
|
||
if (!tags && meta.serverId != null) {
|
||
const [row] = await db.select({ tags: servers.tags }).from(servers).where(inArray(servers.id, [meta.serverId]));
|
||
tags = parseTags(row?.tags);
|
||
}
|
||
channels = firstMatchingRule(routing.rules, { ...meta, tags: tags ?? [] })?.channels ?? null;
|
||
}
|
||
let pushIds: number[] | null = null;
|
||
if (channels === null || channels.includes("webPush")) {
|
||
const subs = await db.select({ id: pushSubscriptions.id, minSeverity: pushSubscriptions.minSeverity, categories: pushSubscriptions.categories }).from(pushSubscriptions);
|
||
// Only when somebody has said what they want; otherwise everyone gets it, as ever.
|
||
if (subs.some((s) => s.minSeverity || s.categories)) pushIds = subs.filter((s) => pushAccepts(s, meta)).map((s) => s.id);
|
||
}
|
||
return { channels, pushIds };
|
||
}
|
||
|
||
async function sendToChannels(title: string, message: string, audience: Audience = EVERYONE, actions: NotifyActions | null = null): Promise<void> {
|
||
const to = (c: ChannelId) => audience.channels === null || audience.channels.includes(c);
|
||
await Promise.all([
|
||
to("gotify") ? sendGotify(title, message, actions) : undefined,
|
||
to("ntfy") ? sendNtfy(title, message, actions) : undefined,
|
||
to("telegram") ? sendTelegram(title, message, actions) : undefined,
|
||
to("pushover") ? sendPushover(title, message, actions) : undefined,
|
||
to("smtp") ? sendSmtp(title, message, actions) : undefined,
|
||
to("webhook") ? sendWebhook(title, message, actions) : undefined,
|
||
to("webPush") ? sendWebPush(title, message, audience.pushIds, actions) : undefined,
|
||
audience.email ? sendExtraEmail(title, actions ? message + footerText(actions) : message, audience.email) : undefined,
|
||
]);
|
||
}
|
||
|
||
/**
|
||
* Sends to an audience that is already worked out: held for the quiet-hours digest if it is quiet (unless told not to), nowhere if a rule
|
||
* said nobody. Escalations use this with their own channels.
|
||
*/
|
||
export async function notifyAudience(title: string, message: string, audience: Audience, options: { quietHours?: boolean; actions?: NotifyActions | null; critical?: boolean } = {}): Promise<void> {
|
||
if (isMuted(audience)) return;
|
||
// Held when quiet hours or an away stretch say so — unless it is critical and they let critical through (services/availability.ts).
|
||
if (options.quietHours !== false && (await holdReason(options.critical === true))) {
|
||
await enqueueNotification(title, message, audience.channels === null && audience.pushIds === null && !audience.email ? null : JSON.stringify(audience));
|
||
return;
|
||
}
|
||
await sendToChannels(title, message, audience, options.actions ?? null);
|
||
}
|
||
|
||
/** Items that may each be for a different audience (a rule can send a disk warning one place and a server outage another): sent as one message per audience. */
|
||
async function notifyGrouped<T>(items: T[], metaOf: (item: T) => NotifyMeta, build: (group: T[]) => { title: string; message: string }, keysOf?: (item: T) => string | undefined): Promise<void> {
|
||
const groups = new Map<string, { audience: Audience; items: T[] }>();
|
||
for (const item of items) {
|
||
const audience = await audienceFor(metaOf(item));
|
||
const key = audienceKey(audience);
|
||
const group = groups.get(key) ?? { audience, items: [] };
|
||
group.items.push(item);
|
||
groups.set(key, group);
|
||
}
|
||
for (const group of groups.values()) {
|
||
const { title, message } = build(group.items);
|
||
// Buttons, when the message is about alerts that have a key and an administrator has switched them on (a message held for quiet hours is sent without them).
|
||
const critical = group.items.some((i) => metaOf(i).severity === "critical");
|
||
await notifyAudience(title, message, group.audience, { critical, ...(keysOf ? { actions: await actionsFor(group.items.map(keysOf)) } : {}) });
|
||
}
|
||
}
|
||
|
||
/** Sends at once through every channel that is switched on, whatever the time — for something a person just asked for. */
|
||
export async function notifyNow(title: string, message: string): Promise<void> {
|
||
await sendToChannels(title, message);
|
||
}
|
||
|
||
/** How many channels are switched on and filled in enough to send, so a "send it now" can say when there is nowhere to send to. */
|
||
export async function enabledChannelCount(): Promise<number> {
|
||
const { gotify, ntfy, telegram, pushover, smtp, webhook, webPush } = await getSettings();
|
||
return [
|
||
webPush.enabled && (await subscriptionCount()) > 0,
|
||
gotify.enabled && gotify.url && gotify.token,
|
||
ntfy.enabled && ntfy.url && ntfy.topic,
|
||
telegram.enabled && telegram.token && telegram.chatId,
|
||
pushover.enabled && pushover.appToken && pushover.userKey,
|
||
smtp.enabled && smtp.host && smtp.to && smtp.from,
|
||
webhook.enabled && webhook.url,
|
||
].filter(Boolean).length;
|
||
}
|
||
|
||
/** `meta` says what it is about, so routing rules and browsers' wishes can apply; without it, it goes to everyone. */
|
||
export async function notify(title: string, message: string, meta?: NotifyMeta): Promise<void> {
|
||
await notifyAudience(title, message, await audienceFor(meta), { critical: meta?.severity === "critical" });
|
||
}
|
||
|
||
/**
|
||
* Sends everything held as one digest per audience, then clears the queue: called when quiet hours end, when an away stretch ends, or by hand (`force`). What is held for one
|
||
* reason is not let out while the other still holds it: quiet hours ending in the middle of a holiday release nothing, and coming back at 02:00 releases nothing until morning.
|
||
*/
|
||
export async function flushQuietHoursQueue(options: { reason?: "quiet" | "away"; force?: boolean } = {}): Promise<number> {
|
||
const reason = options.reason ?? "quiet";
|
||
if (!options.force) {
|
||
const s = await getSettings();
|
||
const held = holdFor({ away: s.away, quietHours: s.quietHours, timezone: s.notifications.timezone }, false, new Date());
|
||
if (held !== null) return 0;
|
||
}
|
||
const queued = await listQueuedNotifications();
|
||
if (queued.length === 0) return 0;
|
||
// One digest for each audience the held notifications were for.
|
||
const groups = new Map<string, typeof queued>();
|
||
for (const q of queued) groups.set(q.audience ?? "", [...(groups.get(q.audience ?? "") ?? []), q]);
|
||
for (const [stored, items] of groups) {
|
||
let audience: Audience = EVERYONE;
|
||
try {
|
||
if (stored) audience = JSON.parse(stored) as Audience;
|
||
} catch {
|
||
// unreadable: everyone, as before
|
||
}
|
||
const sections = items.map((q) => `— ${q.title} —\n${q.message}`);
|
||
await sendToChannels(
|
||
reason === "away" ? "Homelab Manager — Notifications from while you were away" : "Homelab Manager — Notifications from quiet hours",
|
||
`${items.length} notification${items.length !== 1 ? "s were" : " was"} held ${reason === "away" ? "while you were away" : "during quiet hours"}:\n\n${sections.join("\n\n")}`,
|
||
audience,
|
||
);
|
||
}
|
||
await clearQueuedNotifications();
|
||
return queued.length;
|
||
}
|
||
|
||
// ─── Test senders — throw on failure so the route can report it ────────────
|
||
|
||
export async function testGotify(cfg: { url: string; token: string; priority?: number }): Promise<void> {
|
||
const base = cfg.url.replace(/\/$/, "");
|
||
const res = await fetch(`${base}/message?token=${encodeURIComponent(cfg.token)}`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
title: "Homelab Manager — Test",
|
||
message: "Gotify notifications are working correctly.",
|
||
priority: cfg.priority ?? 5,
|
||
}),
|
||
});
|
||
if (!res.ok) throw new Error(`Gotify returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise<void> {
|
||
const base = cfg.url.replace(/\/$/, "");
|
||
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: headerSafe("Homelab Manager — Test"), Priority: String(cfg.priority ?? 3) };
|
||
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
|
||
const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, {
|
||
method: "POST",
|
||
headers,
|
||
body: "ntfy notifications are working correctly.",
|
||
});
|
||
if (!res.ok) throw new Error(`ntfy returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
export async function testTelegram(cfg: { token: string; chatId: string; silent?: boolean }): Promise<void> {
|
||
await telegramSend(cfg, "Homelab Manager — Test\n\nTelegram notifications are working correctly.");
|
||
}
|
||
|
||
export async function testPushover(cfg: { appToken: string; userKey: string; device?: string; priority?: number }): Promise<void> {
|
||
await pushoverSend(cfg, "Homelab Manager — Test", "Pushover notifications are working correctly.");
|
||
}
|
||
|
||
export async function testSmtp(cfg: {
|
||
host: string;
|
||
port?: number;
|
||
secure?: boolean;
|
||
username?: string;
|
||
password?: string;
|
||
from: string;
|
||
to: string;
|
||
}): Promise<void> {
|
||
const transporter = nodemailer.createTransport({
|
||
host: cfg.host,
|
||
port: Number(cfg.port) || 587,
|
||
secure: !!cfg.secure,
|
||
auth: cfg.username ? { user: cfg.username, pass: cfg.password } : undefined,
|
||
});
|
||
await transporter.verify();
|
||
await transporter.sendMail({ from: cfg.from, to: cfg.to, subject: "Homelab Manager — Test", text: "SMTP notifications are working correctly." });
|
||
}
|
||
|
||
export async function testWebhook(cfg: { url: string; secret?: string }): Promise<void> {
|
||
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
||
if (cfg.secret) headers["X-Webhook-Secret"] = cfg.secret;
|
||
const res = await fetch(cfg.url, {
|
||
method: "POST",
|
||
headers,
|
||
body: JSON.stringify({ content: "**Homelab Manager — Test**\nWebhook notifications are working correctly." }),
|
||
});
|
||
if (!res.ok) throw new Error(`Webhook returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
// ─── Event helpers ───────────────────────────────────────────────────────────
|
||
|
||
async function eventEnabled(key: keyof NotificationEvents): Promise<boolean> {
|
||
const { notifications } = await getSettings();
|
||
return notifications[key] !== false;
|
||
}
|
||
|
||
export async function notifyDnsRecordAdded(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsAdd"))) return;
|
||
await notify("DNS Record Added", `[${providerLabel}] ${zoneName}\n+ ${record.type} ${record.name} → ${record.content}`);
|
||
}
|
||
|
||
export async function notifyDnsRecordUpdated(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsUpdate"))) return;
|
||
await notify("DNS Record Updated", `[${providerLabel}] ${zoneName}\n✎ ${record.type} ${record.name} → ${record.content}`);
|
||
}
|
||
|
||
export async function notifyDnsRecordDeleted(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsDelete"))) return;
|
||
await notify("DNS Record Deleted", `[${providerLabel}] ${zoneName}\n− ${record.type} ${record.name} ${record.content}`);
|
||
}
|
||
|
||
export async function notifySecretExpiry(
|
||
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number }[],
|
||
checkFailures: { name: string; host: string; port: number; error?: string }[] = [],
|
||
): Promise<void> {
|
||
if (expiring.length === 0 && checkFailures.length === 0) return;
|
||
if (!(await eventEnabled("secretCheck"))) return;
|
||
const sections: string[] = [];
|
||
if (expiring.length > 0) {
|
||
const lines = expiring.map((s) => (s.status === "expired" ? `✕ EXPIRED — ${s.name}` : `⚠ ${s.daysLeft}d left — ${s.name}`));
|
||
sections.push(`${expiring.length} secret${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`);
|
||
}
|
||
if (checkFailures.length > 0) {
|
||
const lines = checkFailures.map((f) => `⚠ ${f.name} (${f.host}:${f.port}) — ${f.error ?? "check failed"}`);
|
||
sections.push(
|
||
`Couldn't read the live certificate for ${checkFailures.length} monitored secret${checkFailures.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"), { category: "expiry", severity: expiring.some((s) => s.status === "expired") ? "critical" : "warning" });
|
||
}
|
||
|
||
export async function notifyDockerUpdates(
|
||
updatesAvailable: { integrationName: string; containerName: string; environmentName: string; newerVersion: string | null }[],
|
||
): Promise<void> {
|
||
if (updatesAvailable.length === 0) return;
|
||
if (!(await eventEnabled("dockerUpdateCheck"))) return;
|
||
const lines = updatesAvailable.map(
|
||
(u) => `${u.containerName} [${u.environmentName}, ${u.integrationName}]${u.newerVersion ? ` → ${u.newerVersion}` : ""}`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Docker Updates Available",
|
||
`${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`,
|
||
{ category: "updates", severity: "warning" },
|
||
);
|
||
}
|
||
|
||
export async function notifyProxmoxBackupFailure(
|
||
failures: { integrationName: string; node: string; guestId: string | null; status: string }[],
|
||
): Promise<void> {
|
||
if (failures.length === 0) return;
|
||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||
const lines = failures.map(
|
||
(f) => `${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} [${f.integrationName}]: ${f.status}`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Proxmox Backup Failed",
|
||
`${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
|
||
{ category: "backup", severity: "warning" },
|
||
);
|
||
}
|
||
|
||
export async function notifyProxmoxUncoveredGuests(
|
||
uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[],
|
||
): Promise<void> {
|
||
if (uncovered.length === 0) return;
|
||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||
const lines = uncovered.map((u) => `${u.guestName} (#${u.vmid}) on ${u.node} [${u.integrationName}]`);
|
||
await notify(
|
||
"Homelab Manager — Proxmox Guests Without Backup Coverage",
|
||
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
|
||
{ category: "backup", severity: "warning" },
|
||
);
|
||
}
|
||
|
||
export async function notifyStaleBackups(stale: { message: string }[]): Promise<void> {
|
||
if (stale.length === 0) return;
|
||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||
await notify(
|
||
"Homelab Manager — Backups Gone Stale",
|
||
`${stale.length} guest${stale.length !== 1 ? "s have" : " has"} no recent backup:\n\n${stale.map((s) => s.message).join("\n")}`,
|
||
{ category: "backup", severity: "warning" },
|
||
);
|
||
}
|
||
|
||
export async function notifyUpdateRunFinished(run: { serverName: string; templateName: string; ok: boolean; message: string }): Promise<void> {
|
||
if (!(await eventEnabled("updateRunResults"))) return;
|
||
await notify(
|
||
run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`,
|
||
`"${run.templateName}" on ${run.serverName}: ${run.message}`,
|
||
{ category: "updates", severity: run.ok ? "warning" : "critical" },
|
||
);
|
||
}
|
||
|
||
/** How a patch plan's run ended: one message, with a line for each server. Held back or sent with the other update-run results. */
|
||
export async function notifyPatchPlanFinished(run: { planName: string; ok: boolean; message: string; lines: string[] }): Promise<void> {
|
||
if (!(await eventEnabled("updateRunResults"))) return;
|
||
await notify(
|
||
run.ok ? `Homelab Manager — Patch plan "${run.planName}" finished` : `Homelab Manager — Patch plan "${run.planName}" didn't finish cleanly`,
|
||
`${run.message}${run.lines.length > 0 ? `\n\n${run.lines.join("\n")}` : ""}`,
|
||
{ category: "updates", severity: run.ok ? "warning" : "critical" },
|
||
);
|
||
}
|
||
|
||
export async function notifyPbsVerificationFailed(
|
||
failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[],
|
||
): Promise<void> {
|
||
if (failures.length === 0) return;
|
||
if (!(await eventEnabled("pbsVerificationCheck"))) return;
|
||
const lines = failures.map((f) =>
|
||
f.error
|
||
? `${f.datastore} [${f.integrationName}]: couldn't be read — ${f.error}`
|
||
: `${f.datastore} [${f.integrationName}]: ${f.failedCount} snapshot${f.failedCount !== 1 ? "s" : ""} failed verification`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Proxmox Backup Server Verification Failed",
|
||
`${failures.length} datastore${failures.length !== 1 ? "s have" : " has"} a problem:\n\n${lines.join("\n")}`,
|
||
{ category: "backup", severity: "warning" },
|
||
);
|
||
}
|
||
|
||
/** What a problem is about, for routing: its key says the kind and the server; the rest follows its severity. */
|
||
const issueMeta = (i: { key?: string; severity?: string }): NotifyMeta => (i.key ? metaOfConditionKey(i.key, i.severity) : { category: null, severity: i.severity === "critical" ? "critical" : "warning" });
|
||
|
||
export async function notifyHealthIssues(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
|
||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||
if (issues.length === 0) return;
|
||
// One somebody has snoozed is not announced now; if it is still there when the snooze ends, it is announced then (services/alertSnoozes.ts).
|
||
issues = await withoutSnoozed(issues);
|
||
if (issues.length === 0) return;
|
||
// An alert that keeps coming and going is announced once as "flapping" and then left quiet (services/flapGuard.ts); that is tracked whether or not this is switched on.
|
||
const { pass, started } = await dampRaised(issues);
|
||
if (!(await eventEnabled("healthAlerts"))) return;
|
||
if (started.length > 0) await notifyFlapping(started);
|
||
if (pass.length === 0) return;
|
||
issues = pass;
|
||
await notifyGrouped(issues, issueMeta, (group) => ({
|
||
title: "Homelab Manager — Health Alert",
|
||
message: `${group.length} new issue${group.length !== 1 ? "s" : ""}:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`,
|
||
}), (i) => i.key);
|
||
}
|
||
|
||
export async function notifyHealthRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||
if (resolved.length === 0) return;
|
||
resolved = await withoutSnoozed(resolved);
|
||
if (resolved.length === 0) return;
|
||
const passed = await dampRecovered(resolved);
|
||
if (!(await eventEnabled("healthAlerts"))) return;
|
||
if (passed.length === 0) return;
|
||
resolved = passed;
|
||
await notifyGrouped(resolved, issueMeta, (group) => ({
|
||
title: "Homelab Manager — Health Recovered",
|
||
message: `${group.length} issue${group.length !== 1 ? "s" : ""} cleared:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||
}));
|
||
}
|
||
|
||
/** Alerts that were snoozed, whose time is up and that are still there: said once, as a reminder. */
|
||
export async function notifySnoozeEnded(items: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||
if (items.length === 0) return;
|
||
await notifyGrouped(items, issueMeta, (group) => ({
|
||
title: "Homelab Manager — Snooze ended",
|
||
message: `${group.length === 1 ? "An alert you snoozed is" : `${group.length} alerts you snoozed are`} still there:\n\n${group.map((i) => `⏰ ${i.message}`).join("\n")}`,
|
||
}), (i) => i.key);
|
||
}
|
||
|
||
/** Alerts that somebody closed and that have since got more serious: opened again, and said once. */
|
||
export async function notifyClosedWorse(items: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||
if (items.length === 0) return;
|
||
await notifyGrouped(items, issueMeta, (group) => ({
|
||
title: "Homelab Manager — Closed alert got worse",
|
||
message: `${group.length === 1 ? "An alert you closed is" : `${group.length} alerts you closed are`} more serious now, so ${group.length === 1 ? "it is" : "they are"} open again:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`,
|
||
}), (i) => i.key);
|
||
}
|
||
|
||
/** An alert that has just been called flapping: said once, with how often it has come, instead of each time. */
|
||
export async function notifyFlapping(items: { message: string; key?: string; severity?: string; count: number }[]): Promise<void> {
|
||
if (items.length === 0) return;
|
||
await notifyGrouped(items, issueMeta, (group) => ({
|
||
title: "Homelab Manager — Flapping alert",
|
||
message: `${group.length === 1 ? "An alert keeps" : `${group.length} alerts keep`} coming and going:\n\n${group.map((i) => `↻ ${i.message} — raised ${i.count} times within an hour`).join("\n")}\n\nIts further messages are held back until it has been quiet for an hour; it stays on the Alerts page, marked as flapping.`,
|
||
}), (i) => i.key);
|
||
}
|
||
|
||
/** Flapping alerts that have been quiet for an hour: what state each is in now. */
|
||
export async function notifySettled(items: { message: string; key: string; active: boolean }[]): Promise<void> {
|
||
if (items.length === 0 || !(await eventEnabled("healthAlerts"))) return;
|
||
await notifyGrouped(items, issueMeta, (group) => ({
|
||
title: "Homelab Manager — Alert settled",
|
||
message: `${group.length === 1 ? "An alert that was flapping has" : `${group.length} alerts that were flapping have`} been steady for an hour:\n\n${group.map((i) => `${i.active ? "⚠ still active" : "✓ cleared"} — ${i.message}`).join("\n")}`,
|
||
}));
|
||
}
|
||
|
||
export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
|
||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||
if (issues.length === 0) return;
|
||
issues = await withoutSnoozed(issues);
|
||
if (issues.length === 0 || !(await eventEnabled("automationAlerts"))) return;
|
||
await notifyGrouped(issues, issueMeta, (group) => ({
|
||
title: "Homelab Manager — Automation Failed",
|
||
message: `${group.length} run${group.length !== 1 ? "s" : ""} failed:\n\n${group.map((i) => `✕ ${i.message}`).join("\n")}`,
|
||
}), (i) => i.key);
|
||
}
|
||
|
||
export async function notifyAutomationRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
|
||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||
if (resolved.length === 0) return;
|
||
resolved = await withoutSnoozed(resolved);
|
||
if (resolved.length === 0 || !(await eventEnabled("automationAlerts"))) return;
|
||
await notifyGrouped(resolved, issueMeta, (group) => ({
|
||
title: "Homelab Manager — Automation Recovered",
|
||
message: `${group.length} no longer failing:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||
}));
|
||
}
|
||
|
||
export async function notifyDomainExpiry(
|
||
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number; expiresAt: string; registrar: string | null }[],
|
||
staleChecks: { name: string; error: string; lastOkAt: string | null }[] = [],
|
||
): Promise<void> {
|
||
if (expiring.length === 0 && staleChecks.length === 0) return;
|
||
if (!(await eventEnabled("domainExpiryCheck"))) return;
|
||
const sections: string[] = [];
|
||
if (expiring.length > 0) {
|
||
const lines = expiring.map((d) => {
|
||
const registrar = d.registrar ? ` [${d.registrar}]` : "";
|
||
return d.status === "expired"
|
||
? `✕ EXPIRED ${d.expiresAt} — ${d.name}${registrar}`
|
||
: `⚠ ${d.daysLeft}d left (${d.expiresAt}) — ${d.name}${registrar}`;
|
||
});
|
||
sections.push(`${expiring.length} domain${expiring.length !== 1 ? "s" : ""} need renewing:\n\n${lines.join("\n")}`);
|
||
}
|
||
if (staleChecks.length > 0) {
|
||
const lines = staleChecks.map((s) => `⚠ ${s.name} — ${s.error}${s.lastOkAt ? ` (last read ${s.lastOkAt.slice(0, 10)})` : " (never read)"}`);
|
||
sections.push(
|
||
`Couldn't refresh the registration for ${staleChecks.length} domain${staleChecks.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"), { category: "expiry", severity: expiring.some((d) => d.status === "expired") ? "critical" : "warning" });
|
||
}
|
||
|
||
export async function notifyTailscaleKeyExpiry(
|
||
expiring: { integrationName: string; deviceLabel: string; daysLeft: number }[],
|
||
): Promise<void> {
|
||
if (expiring.length === 0) return;
|
||
if (!(await eventEnabled("tailscaleKeyCheck"))) return;
|
||
const lines = expiring.map(
|
||
(d) => `${d.daysLeft < 0 ? "✕ EXPIRED" : `⚠ ${d.daysLeft}d left`} — ${d.deviceLabel} [${d.integrationName}]`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Tailscale Key Expiry",
|
||
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
|
||
{ category: "expiry", severity: expiring.some((d) => d.daysLeft < 0) ? "critical" : "warning" },
|
||
);
|
||
}
|
||
|
||
// Diagnostic-log "source" values (see integrations/*/adapter.ts and dns/adapters/*.ts) to display names.
|
||
const SOURCE_LABELS: Record<string, string> = {
|
||
cloudflare: "Cloudflare",
|
||
loopia: "Loopia",
|
||
pihole: "Pi-hole",
|
||
azure: "Azure DNS",
|
||
cpanel: "cPanel",
|
||
technitium: "Technitium",
|
||
tailscale: "Tailscale",
|
||
proxmox: "Proxmox",
|
||
synology: "Synology",
|
||
semaphore: "Semaphore",
|
||
gitea: "Gitea",
|
||
dockhand: "Dockhand",
|
||
uptimekuma: "Uptime Kuma",
|
||
phpipam: "phpIPAM",
|
||
pbs: "Proxmox Backup Server",
|
||
osticket: "osTicket",
|
||
unifi: "UniFi",
|
||
npm: "Nginx Proxy Manager",
|
||
homeassistant: "Home Assistant",
|
||
};
|
||
|
||
export function sourceLabel(source: string): string {
|
||
return SOURCE_LABELS[source] ?? source;
|
||
}
|
||
|
||
export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise<void> {
|
||
emitEvent("alert.opened", { alert: { key: `integration:${source}`, source, severity: "warning", message: `${sourceLabel(source)} has failed its last ${consecutiveFailures} calls in a row.` } });
|
||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Integration Down",
|
||
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
|
||
{ category: "integration", severity: "warning" },
|
||
);
|
||
}
|
||
|
||
export async function notifyIntegrationRecovered(source: string): Promise<void> {
|
||
emitEvent("alert.resolved", { alert: { key: `integration:${source}`, message: `${sourceLabel(source)} is answering again.` } });
|
||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`, { category: "integration", severity: "warning" });
|
||
}
|