Files
Homelab-manager/server/src/services/notify.ts
T
bobbanandClaude Sonnet 5.5 31f4debb9a Add Telegram and Pushover as notification channels
Both send to one fixed address each (api.telegram.org, api.pushover.net), take part in routing rules and the channel count, have a Send test that shows the service's own refusal, and keep their tokens and keys encrypted and out of errors and the audit log. A Telegram message carries one button, and a Pushover message one link, to the page where an alert can be acknowledged, silenced or snoozed.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-05 00:30:29 +01:00

743 lines
39 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import nodemailer from "nodemailer";
import { getSettings, type NotificationEvents } from "./settingsStore.js";
import { holdFor, holdReason } from "./availability.js";
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
import { emitEvent } from "./eventWebhooks.js";
import { sendPush, subscriptionCount } from "./webPush.js";
import { db } from "../db/client.js";
import { inArray } from "drizzle-orm";
import { pushSubscriptions, servers } from "../db/schema.js";
import { parseTags } from "./serverTags.js";
import { dampRaised, dampRecovered } from "./flapGuard.js";
import { withoutSnoozed } from "./alertSnoozes.js";
import { actionsFor, footerText, ntfyActionsHeader, type NotifyActions } from "./alertActionLinks.js";
import { firstMatchingRule, metaOfConditionKey, pushAccepts, type ChannelId, type NotifyMeta } from "./notifyRouting.js";
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
async function sendGotify(title: string, message: string, actions?: NotifyActions | null) {
const { gotify } = await getSettings();
if (!gotify.enabled || !gotify.url || !gotify.token) return;
const base = gotify.url.replace(/\/$/, "");
try {
const res = await fetch(`${base}/message?token=${encodeURIComponent(gotify.token)}`, {
method: "POST",
headers: { "Content-Type": "application/json" },
// Gotify has no buttons: a tap opens the page with them (and the address is in the text, for a client that does not tap).
body: JSON.stringify({ title, message: actions ? message + footerText(actions) : message, priority: gotify.priority ?? 5, ...(actions ? { extras: { "client::notification": { click: { url: actions.pageUrl } } } } : {}) }),
});
if (!res.ok) console.error(`[notify] Gotify error ${res.status}: ${await res.text().catch(() => "")}`);
} catch (err) {
console.error("[notify] Gotify failed:", err instanceof Error ? err.message : err);
}
}
/**
* A header can only carry Latin-1, and every title here has an em dash. ntfy reads RFC 2047 encoded words in its headers ("=?UTF-8?B?...?="),
* so anything outside plain ASCII is sent that way.
*/
export function headerSafe(text: string): string {
// eslint-disable-next-line no-control-regex
return /^[ -~]*$/.test(text) ? text : `=?UTF-8?B?${Buffer.from(text, "utf8").toString("base64")}?=`;
}
async function sendNtfy(title: string, message: string, actions?: NotifyActions | null) {
const { ntfy } = await getSettings();
if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return;
const base = ntfy.url.replace(/\/$/, "");
const headers: Record<string, string> = {
"Content-Type": "text/plain",
Title: headerSafe(title),
Priority: String(ntfy.priority ?? 3),
};
if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`;
if (actions) {
headers.Actions = ntfyActionsHeader(actions);
headers.Click = actions.pageUrl;
}
try {
const res = await fetch(`${base}/${encodeURIComponent(ntfy.topic)}`, { method: "POST", headers, body: message });
if (!res.ok) console.error(`[notify] ntfy error ${res.status}: ${await res.text().catch(() => "")}`);
} catch (err) {
console.error("[notify] ntfy failed:", err instanceof Error ? err.message : err);
}
}
// ─── Telegram and Pushover ──────────────────────────────────────────────────
//
// Both are hosted services with one fixed address each (so nothing a person types decides where a request goes, as it does for a webhook). The addresses are kept here, in one object, so a test can
// point them at a pretend server. Neither says why it refused in its HTTP status alone: Telegram answers { ok: false, description }, Pushover { status: 0, errors: [...] }, and that is what is shown.
export const CHANNEL_ENDPOINTS = { telegram: "https://api.telegram.org", pushover: "https://api.pushover.net" };
const clip = (text: string, max: number) => (text.length <= max ? text : `${text.slice(0, max - 1)}…`);
/** Telegram allows 4096 characters in a message. */
export const TELEGRAM_MAX = 4000;
/** Pushover allows 1024 in a message, 250 in a title, 512 in a link. */
export const PUSHOVER_MESSAGE_MAX = 1024;
export const PUSHOVER_TITLE_MAX = 250;
export const PUSHOVER_URL_MAX = 512;
const BUTTON_LABEL = "Acknowledge, silence or snooze";
/** Sends one message through a Telegram bot; throws what Telegram says when it refuses. The bot's token is in the address, so it is never put in an error. */
export async function telegramSend(cfg: { token: string; chatId: string; silent?: boolean }, text: string, actions?: NotifyActions | null): Promise<void> {
const post = async (body: Record<string, unknown>) => {
const res = await fetch(`${CHANNEL_ENDPOINTS.telegram}/bot${cfg.token}/sendMessage`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) });
const json = (await res.json().catch(() => null)) as { ok?: boolean; description?: string } | null;
return { ok: res.ok && json?.ok !== false, status: res.status, description: json?.description ?? null };
};
const base = { chat_id: cfg.chatId, disable_web_page_preview: true, ...(cfg.silent ? { disable_notification: true } : {}) };
// A button that opens the page where the alert can be dealt with — plain text for what is said, so nothing in it needs escaping.
let result = await post({ ...base, text: clip(text, TELEGRAM_MAX), ...(actions ? { reply_markup: { inline_keyboard: [[{ text: BUTTON_LABEL, url: actions.pageUrl }]] } } : {}) });
// Telegram refuses a button whose address it cannot open (an address on a private network, say): say it in the text instead.
if (!result.ok && actions && result.status === 400) result = await post({ ...base, text: clip(text + footerText(actions), TELEGRAM_MAX) });
if (!result.ok) throw new Error(`Telegram: ${result.description ?? `HTTP ${result.status}`}`);
}
/** Sends one message through Pushover; throws what Pushover says when it refuses. */
export async function pushoverSend(cfg: { appToken: string; userKey: string; device?: string; priority?: number }, title: string, message: string, actions?: NotifyActions | null): Promise<void> {
const priority = Number.isInteger(cfg.priority) ? Math.min(2, Math.max(-2, cfg.priority as number)) : 0;
const link = actions && actions.pageUrl.length <= PUSHOVER_URL_MAX ? actions.pageUrl : null;
const body: Record<string, string | number> = {
token: cfg.appToken,
user: cfg.userKey,
title: clip(title, PUSHOVER_TITLE_MAX),
// A link too long for Pushover goes in the text instead.
message: clip(actions && !link ? message + footerText(actions) : message, PUSHOVER_MESSAGE_MAX),
priority,
};
if (cfg.device?.trim()) body.device = cfg.device.trim();
// An emergency message repeats until someone acknowledges it: Pushover wants to be told how often (every minute) and for how long (an hour).
if (priority === 2) {
body.retry = 60;
body.expire = 3600;
}
if (link) {
body.url = link;
body.url_title = BUTTON_LABEL;
}
const res = await fetch(`${CHANNEL_ENDPOINTS.pushover}/1/messages.json`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) });
const json = (await res.json().catch(() => null)) as { status?: number; errors?: string[] } | null;
if (!res.ok || json?.status !== 1) throw new Error(`Pushover: ${json?.errors?.length ? json.errors.join(", ") : `HTTP ${res.status}`}`);
}
async function sendTelegram(title: string, message: string, actions?: NotifyActions | null) {
const { telegram } = await getSettings();
if (!telegram.enabled || !telegram.token || !telegram.chatId) return;
try {
await telegramSend(telegram, `${title}\n\n${message}`, actions);
} catch (err) {
console.error("[notify]", err instanceof Error ? err.message : "Telegram failed");
}
}
async function sendPushover(title: string, message: string, actions?: NotifyActions | null) {
const { pushover } = await getSettings();
if (!pushover.enabled || !pushover.appToken || !pushover.userKey) return;
try {
await pushoverSend(pushover, title, message, actions);
} catch (err) {
console.error("[notify]", err instanceof Error ? err.message : "Pushover failed");
}
}
async function sendSmtp(title: string, message: string, actions?: NotifyActions | null) {
const { smtp } = await getSettings();
if (!smtp.enabled || !smtp.host || !smtp.to || !smtp.from) return;
try {
const transporter = nodemailer.createTransport({
host: smtp.host,
port: Number(smtp.port) || 587,
secure: !!smtp.secure,
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
});
await transporter.sendMail({ from: smtp.from, to: smtp.to, subject: title, text: actions ? message + footerText(actions) : message });
} catch (err) {
console.error("[notify] SMTP failed:", err instanceof Error ? err.message : err);
}
}
async function sendWebhook(title: string, message: string, actions?: NotifyActions | null) {
const { webhook } = await getSettings();
if (!webhook.enabled || !webhook.url) return;
const headers: Record<string, string> = { "Content-Type": "application/json" };
if (webhook.secret) headers["X-Webhook-Secret"] = webhook.secret;
try {
const res = await fetch(webhook.url, {
method: "POST",
headers,
body: JSON.stringify({ content: `**${title}**\n${message}${actions ? footerText(actions) : ""}` }),
});
if (!res.ok) console.error(`[notify] Webhook error ${res.status}: ${await res.text().catch(() => "")}`);
} catch (err) {
console.error("[notify] Webhook failed:", err instanceof Error ? err.message : err);
}
}
async function sendWebPush(title: string, message: string, only: number[] | null, actions?: NotifyActions | null) {
const { webPush } = await getSettings();
if (!webPush.enabled) return;
// For one alert, buttons on the notification itself (browsers show two); for several, a tap opens the app.
const buttons = actions?.direct.slice(0, 2).map((d) => ({ action: d.id, title: d.label, path: d.path }));
const message_ = { title, body: message, ...(buttons?.length ? { actions: buttons } : {}) };
try {
if (only === null) await sendPush(message_);
else if (only.length > 0) await sendPush(message_, await db.select().from(pushSubscriptions).where(inArray(pushSubscriptions.id, only)));
} catch (err) {
console.error("[notify] Web push failed:", err instanceof Error ? err.message : err);
}
}
/** Email to one address, through the email channel's server, whatever the channel's own recipient is (an escalation to someone else). */
async function sendExtraEmail(title: string, message: string, to: string) {
const { smtp } = await getSettings();
if (!smtp.host || !smtp.from) {
console.error("[notify] An escalation wants to email someone, but email isn't set up.");
return;
}
try {
const transporter = nodemailer.createTransport({
host: smtp.host,
port: Number(smtp.port) || 587,
secure: !!smtp.secure,
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
});
await transporter.sendMail({ from: smtp.from, to, subject: title, text: message });
} catch (err) {
console.error("[notify] Extra email failed:", err instanceof Error ? err.message : err);
}
}
// ─── Who a notification is for ─────────────────────────────────────────────
/** The channels and browsers a notification goes to. `null` means every one that is on (the way it was before routing existed). */
export interface Audience {
channels: ChannelId[] | null;
/** The subscribed browsers that want it; null = all of them. */
pushIds: number[] | null;
/** Also email this address. */
email?: string;
}
export const EVERYONE: Audience = { channels: null, pushIds: null };
export const audienceKey = (a: Audience) => JSON.stringify({ c: a.channels ? [...a.channels].sort() : null, p: a.pushIds ? [...a.pushIds].sort((x, y) => x - y) : null, e: a.email ?? null });
/** Told to nobody: a rule said so (the Alerts page still shows it). */
const isMuted = (a: Audience) => a.channels !== null && a.channels.length === 0 && !a.email;
/** Works out who a notification is for: the first routing rule it matches picks the channels, and browsers may turn down what they don't want. */
export async function audienceFor(meta?: NotifyMeta): Promise<Audience> {
if (!meta) return EVERYONE;
const { routing } = await getSettings();
let channels: ChannelId[] | null = null;
if (routing.rules.some((r) => r.enabled)) {
let tags = meta.tags;
if (!tags && meta.serverId != null) {
const [row] = await db.select({ tags: servers.tags }).from(servers).where(inArray(servers.id, [meta.serverId]));
tags = parseTags(row?.tags);
}
channels = firstMatchingRule(routing.rules, { ...meta, tags: tags ?? [] })?.channels ?? null;
}
let pushIds: number[] | null = null;
if (channels === null || channels.includes("webPush")) {
const subs = await db.select({ id: pushSubscriptions.id, minSeverity: pushSubscriptions.minSeverity, categories: pushSubscriptions.categories }).from(pushSubscriptions);
// Only when somebody has said what they want; otherwise everyone gets it, as ever.
if (subs.some((s) => s.minSeverity || s.categories)) pushIds = subs.filter((s) => pushAccepts(s, meta)).map((s) => s.id);
}
return { channels, pushIds };
}
async function sendToChannels(title: string, message: string, audience: Audience = EVERYONE, actions: NotifyActions | null = null): Promise<void> {
const to = (c: ChannelId) => audience.channels === null || audience.channels.includes(c);
await Promise.all([
to("gotify") ? sendGotify(title, message, actions) : undefined,
to("ntfy") ? sendNtfy(title, message, actions) : undefined,
to("telegram") ? sendTelegram(title, message, actions) : undefined,
to("pushover") ? sendPushover(title, message, actions) : undefined,
to("smtp") ? sendSmtp(title, message, actions) : undefined,
to("webhook") ? sendWebhook(title, message, actions) : undefined,
to("webPush") ? sendWebPush(title, message, audience.pushIds, actions) : undefined,
audience.email ? sendExtraEmail(title, actions ? message + footerText(actions) : message, audience.email) : undefined,
]);
}
/**
* Sends to an audience that is already worked out: held for the quiet-hours digest if it is quiet (unless told not to), nowhere if a rule
* said nobody. Escalations use this with their own channels.
*/
export async function notifyAudience(title: string, message: string, audience: Audience, options: { quietHours?: boolean; actions?: NotifyActions | null; critical?: boolean } = {}): Promise<void> {
if (isMuted(audience)) return;
// Held when quiet hours or an away stretch say so — unless it is critical and they let critical through (services/availability.ts).
if (options.quietHours !== false && (await holdReason(options.critical === true))) {
await enqueueNotification(title, message, audience.channels === null && audience.pushIds === null && !audience.email ? null : JSON.stringify(audience));
return;
}
await sendToChannels(title, message, audience, options.actions ?? null);
}
/** Items that may each be for a different audience (a rule can send a disk warning one place and a server outage another): sent as one message per audience. */
async function notifyGrouped<T>(items: T[], metaOf: (item: T) => NotifyMeta, build: (group: T[]) => { title: string; message: string }, keysOf?: (item: T) => string | undefined): Promise<void> {
const groups = new Map<string, { audience: Audience; items: T[] }>();
for (const item of items) {
const audience = await audienceFor(metaOf(item));
const key = audienceKey(audience);
const group = groups.get(key) ?? { audience, items: [] };
group.items.push(item);
groups.set(key, group);
}
for (const group of groups.values()) {
const { title, message } = build(group.items);
// Buttons, when the message is about alerts that have a key and an administrator has switched them on (a message held for quiet hours is sent without them).
const critical = group.items.some((i) => metaOf(i).severity === "critical");
await notifyAudience(title, message, group.audience, { critical, ...(keysOf ? { actions: await actionsFor(group.items.map(keysOf)) } : {}) });
}
}
/** Sends at once through every channel that is switched on, whatever the time — for something a person just asked for. */
export async function notifyNow(title: string, message: string): Promise<void> {
await sendToChannels(title, message);
}
/** How many channels are switched on and filled in enough to send, so a "send it now" can say when there is nowhere to send to. */
export async function enabledChannelCount(): Promise<number> {
const { gotify, ntfy, telegram, pushover, smtp, webhook, webPush } = await getSettings();
return [
webPush.enabled && (await subscriptionCount()) > 0,
gotify.enabled && gotify.url && gotify.token,
ntfy.enabled && ntfy.url && ntfy.topic,
telegram.enabled && telegram.token && telegram.chatId,
pushover.enabled && pushover.appToken && pushover.userKey,
smtp.enabled && smtp.host && smtp.to && smtp.from,
webhook.enabled && webhook.url,
].filter(Boolean).length;
}
/** `meta` says what it is about, so routing rules and browsers' wishes can apply; without it, it goes to everyone. */
export async function notify(title: string, message: string, meta?: NotifyMeta): Promise<void> {
await notifyAudience(title, message, await audienceFor(meta), { critical: meta?.severity === "critical" });
}
/**
* Sends everything held as one digest per audience, then clears the queue: called when quiet hours end, when an away stretch ends, or by hand (`force`). What is held for one
* reason is not let out while the other still holds it: quiet hours ending in the middle of a holiday release nothing, and coming back at 02:00 releases nothing until morning.
*/
export async function flushQuietHoursQueue(options: { reason?: "quiet" | "away"; force?: boolean } = {}): Promise<number> {
const reason = options.reason ?? "quiet";
if (!options.force) {
const s = await getSettings();
const held = holdFor({ away: s.away, quietHours: s.quietHours, timezone: s.notifications.timezone }, false, new Date());
if (held !== null) return 0;
}
const queued = await listQueuedNotifications();
if (queued.length === 0) return 0;
// One digest for each audience the held notifications were for.
const groups = new Map<string, typeof queued>();
for (const q of queued) groups.set(q.audience ?? "", [...(groups.get(q.audience ?? "") ?? []), q]);
for (const [stored, items] of groups) {
let audience: Audience = EVERYONE;
try {
if (stored) audience = JSON.parse(stored) as Audience;
} catch {
// unreadable: everyone, as before
}
const sections = items.map((q) => `— ${q.title} —\n${q.message}`);
await sendToChannels(
reason === "away" ? "Homelab Manager — Notifications from while you were away" : "Homelab Manager — Notifications from quiet hours",
`${items.length} notification${items.length !== 1 ? "s were" : " was"} held ${reason === "away" ? "while you were away" : "during quiet hours"}:\n\n${sections.join("\n\n")}`,
audience,
);
}
await clearQueuedNotifications();
return queued.length;
}
// ─── Test senders — throw on failure so the route can report it ────────────
export async function testGotify(cfg: { url: string; token: string; priority?: number }): Promise<void> {
const base = cfg.url.replace(/\/$/, "");
const res = await fetch(`${base}/message?token=${encodeURIComponent(cfg.token)}`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
title: "Homelab Manager — Test",
message: "Gotify notifications are working correctly.",
priority: cfg.priority ?? 5,
}),
});
if (!res.ok) throw new Error(`Gotify returned ${res.status}: ${await res.text().catch(() => "")}`);
}
export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise<void> {
const base = cfg.url.replace(/\/$/, "");
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: headerSafe("Homelab Manager — Test"), Priority: String(cfg.priority ?? 3) };
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, {
method: "POST",
headers,
body: "ntfy notifications are working correctly.",
});
if (!res.ok) throw new Error(`ntfy returned ${res.status}: ${await res.text().catch(() => "")}`);
}
export async function testTelegram(cfg: { token: string; chatId: string; silent?: boolean }): Promise<void> {
await telegramSend(cfg, "Homelab Manager — Test\n\nTelegram notifications are working correctly.");
}
export async function testPushover(cfg: { appToken: string; userKey: string; device?: string; priority?: number }): Promise<void> {
await pushoverSend(cfg, "Homelab Manager — Test", "Pushover notifications are working correctly.");
}
export async function testSmtp(cfg: {
host: string;
port?: number;
secure?: boolean;
username?: string;
password?: string;
from: string;
to: string;
}): Promise<void> {
const transporter = nodemailer.createTransport({
host: cfg.host,
port: Number(cfg.port) || 587,
secure: !!cfg.secure,
auth: cfg.username ? { user: cfg.username, pass: cfg.password } : undefined,
});
await transporter.verify();
await transporter.sendMail({ from: cfg.from, to: cfg.to, subject: "Homelab Manager — Test", text: "SMTP notifications are working correctly." });
}
export async function testWebhook(cfg: { url: string; secret?: string }): Promise<void> {
const headers: Record<string, string> = { "Content-Type": "application/json" };
if (cfg.secret) headers["X-Webhook-Secret"] = cfg.secret;
const res = await fetch(cfg.url, {
method: "POST",
headers,
body: JSON.stringify({ content: "**Homelab Manager — Test**\nWebhook notifications are working correctly." }),
});
if (!res.ok) throw new Error(`Webhook returned ${res.status}: ${await res.text().catch(() => "")}`);
}
// ─── Event helpers ───────────────────────────────────────────────────────────
async function eventEnabled(key: keyof NotificationEvents): Promise<boolean> {
const { notifications } = await getSettings();
return notifications[key] !== false;
}
export async function notifyDnsRecordAdded(
providerLabel: string,
zoneName: string,
record: { type: string; name: string; content: string },
): Promise<void> {
if (!(await eventEnabled("dnsAdd"))) return;
await notify("DNS Record Added", `[${providerLabel}] ${zoneName}\n+ ${record.type} ${record.name} → ${record.content}`);
}
export async function notifyDnsRecordUpdated(
providerLabel: string,
zoneName: string,
record: { type: string; name: string; content: string },
): Promise<void> {
if (!(await eventEnabled("dnsUpdate"))) return;
await notify("DNS Record Updated", `[${providerLabel}] ${zoneName}\n✎ ${record.type} ${record.name} → ${record.content}`);
}
export async function notifyDnsRecordDeleted(
providerLabel: string,
zoneName: string,
record: { type: string; name: string; content: string },
): Promise<void> {
if (!(await eventEnabled("dnsDelete"))) return;
await notify("DNS Record Deleted", `[${providerLabel}] ${zoneName}\n− ${record.type} ${record.name} ${record.content}`);
}
export async function notifySecretExpiry(
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number }[],
checkFailures: { name: string; host: string; port: number; error?: string }[] = [],
): Promise<void> {
if (expiring.length === 0 && checkFailures.length === 0) return;
if (!(await eventEnabled("secretCheck"))) return;
const sections: string[] = [];
if (expiring.length > 0) {
const lines = expiring.map((s) => (s.status === "expired" ? `✕ EXPIRED — ${s.name}` : `⚠ ${s.daysLeft}d left — ${s.name}`));
sections.push(`${expiring.length} secret${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`);
}
if (checkFailures.length > 0) {
const lines = checkFailures.map((f) => `⚠ ${f.name} (${f.host}:${f.port}) — ${f.error ?? "check failed"}`);
sections.push(
`Couldn't read the live certificate for ${checkFailures.length} monitored secret${checkFailures.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
);
}
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"), { category: "expiry", severity: expiring.some((s) => s.status === "expired") ? "critical" : "warning" });
}
export async function notifyDockerUpdates(
updatesAvailable: { integrationName: string; containerName: string; environmentName: string; newerVersion: string | null }[],
): Promise<void> {
if (updatesAvailable.length === 0) return;
if (!(await eventEnabled("dockerUpdateCheck"))) return;
const lines = updatesAvailable.map(
(u) => `${u.containerName} [${u.environmentName}, ${u.integrationName}]${u.newerVersion ? ` → ${u.newerVersion}` : ""}`,
);
await notify(
"Homelab Manager — Docker Updates Available",
`${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`,
{ category: "updates", severity: "warning" },
);
}
export async function notifyProxmoxBackupFailure(
failures: { integrationName: string; node: string; guestId: string | null; status: string }[],
): Promise<void> {
if (failures.length === 0) return;
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
const lines = failures.map(
(f) => `${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} [${f.integrationName}]: ${f.status}`,
);
await notify(
"Homelab Manager — Proxmox Backup Failed",
`${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
{ category: "backup", severity: "warning" },
);
}
export async function notifyProxmoxUncoveredGuests(
uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[],
): Promise<void> {
if (uncovered.length === 0) return;
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
const lines = uncovered.map((u) => `${u.guestName} (#${u.vmid}) on ${u.node} [${u.integrationName}]`);
await notify(
"Homelab Manager — Proxmox Guests Without Backup Coverage",
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
{ category: "backup", severity: "warning" },
);
}
export async function notifyStaleBackups(stale: { message: string }[]): Promise<void> {
if (stale.length === 0) return;
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
await notify(
"Homelab Manager — Backups Gone Stale",
`${stale.length} guest${stale.length !== 1 ? "s have" : " has"} no recent backup:\n\n${stale.map((s) => s.message).join("\n")}`,
{ category: "backup", severity: "warning" },
);
}
export async function notifyUpdateRunFinished(run: { serverName: string; templateName: string; ok: boolean; message: string }): Promise<void> {
if (!(await eventEnabled("updateRunResults"))) return;
await notify(
run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`,
`"${run.templateName}" on ${run.serverName}: ${run.message}`,
{ category: "updates", severity: run.ok ? "warning" : "critical" },
);
}
/** How a patch plan's run ended: one message, with a line for each server. Held back or sent with the other update-run results. */
export async function notifyPatchPlanFinished(run: { planName: string; ok: boolean; message: string; lines: string[] }): Promise<void> {
if (!(await eventEnabled("updateRunResults"))) return;
await notify(
run.ok ? `Homelab Manager — Patch plan "${run.planName}" finished` : `Homelab Manager — Patch plan "${run.planName}" didn't finish cleanly`,
`${run.message}${run.lines.length > 0 ? `\n\n${run.lines.join("\n")}` : ""}`,
{ category: "updates", severity: run.ok ? "warning" : "critical" },
);
}
export async function notifyPbsVerificationFailed(
failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[],
): Promise<void> {
if (failures.length === 0) return;
if (!(await eventEnabled("pbsVerificationCheck"))) return;
const lines = failures.map((f) =>
f.error
? `${f.datastore} [${f.integrationName}]: couldn't be read — ${f.error}`
: `${f.datastore} [${f.integrationName}]: ${f.failedCount} snapshot${f.failedCount !== 1 ? "s" : ""} failed verification`,
);
await notify(
"Homelab Manager — Proxmox Backup Server Verification Failed",
`${failures.length} datastore${failures.length !== 1 ? "s have" : " has"} a problem:\n\n${lines.join("\n")}`,
{ category: "backup", severity: "warning" },
);
}
/** What a problem is about, for routing: its key says the kind and the server; the rest follows its severity. */
const issueMeta = (i: { key?: string; severity?: string }): NotifyMeta => (i.key ? metaOfConditionKey(i.key, i.severity) : { category: null, severity: i.severity === "critical" ? "critical" : "warning" });
export async function notifyHealthIssues(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
// One somebody has snoozed is not announced now; if it is still there when the snooze ends, it is announced then (services/alertSnoozes.ts).
issues = await withoutSnoozed(issues);
if (issues.length === 0) return;
// An alert that keeps coming and going is announced once as "flapping" and then left quiet (services/flapGuard.ts); that is tracked whether or not this is switched on.
const { pass, started } = await dampRaised(issues);
if (!(await eventEnabled("healthAlerts"))) return;
if (started.length > 0) await notifyFlapping(started);
if (pass.length === 0) return;
issues = pass;
await notifyGrouped(issues, issueMeta, (group) => ({
title: "Homelab Manager — Health Alert",
message: `${group.length} new issue${group.length !== 1 ? "s" : ""}:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`,
}), (i) => i.key);
}
export async function notifyHealthRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
resolved = await withoutSnoozed(resolved);
if (resolved.length === 0) return;
const passed = await dampRecovered(resolved);
if (!(await eventEnabled("healthAlerts"))) return;
if (passed.length === 0) return;
resolved = passed;
await notifyGrouped(resolved, issueMeta, (group) => ({
title: "Homelab Manager — Health Recovered",
message: `${group.length} issue${group.length !== 1 ? "s" : ""} cleared:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
}));
}
/** Alerts that were snoozed, whose time is up and that are still there: said once, as a reminder. */
export async function notifySnoozeEnded(items: { message: string; key?: string; severity?: string }[]): Promise<void> {
if (items.length === 0) return;
await notifyGrouped(items, issueMeta, (group) => ({
title: "Homelab Manager — Snooze ended",
message: `${group.length === 1 ? "An alert you snoozed is" : `${group.length} alerts you snoozed are`} still there:\n\n${group.map((i) => `⏰ ${i.message}`).join("\n")}`,
}), (i) => i.key);
}
/** Alerts that somebody closed and that have since got more serious: opened again, and said once. */
export async function notifyClosedWorse(items: { message: string; key?: string; severity?: string }[]): Promise<void> {
if (items.length === 0) return;
await notifyGrouped(items, issueMeta, (group) => ({
title: "Homelab Manager — Closed alert got worse",
message: `${group.length === 1 ? "An alert you closed is" : `${group.length} alerts you closed are`} more serious now, so ${group.length === 1 ? "it is" : "they are"} open again:\n\n${group.map((i) => `⚠ ${i.message}`).join("\n")}`,
}), (i) => i.key);
}
/** An alert that has just been called flapping: said once, with how often it has come, instead of each time. */
export async function notifyFlapping(items: { message: string; key?: string; severity?: string; count: number }[]): Promise<void> {
if (items.length === 0) return;
await notifyGrouped(items, issueMeta, (group) => ({
title: "Homelab Manager — Flapping alert",
message: `${group.length === 1 ? "An alert keeps" : `${group.length} alerts keep`} coming and going:\n\n${group.map((i) => `↻ ${i.message} — raised ${i.count} times within an hour`).join("\n")}\n\nIts further messages are held back until it has been quiet for an hour; it stays on the Alerts page, marked as flapping.`,
}), (i) => i.key);
}
/** Flapping alerts that have been quiet for an hour: what state each is in now. */
export async function notifySettled(items: { message: string; key: string; active: boolean }[]): Promise<void> {
if (items.length === 0 || !(await eventEnabled("healthAlerts"))) return;
await notifyGrouped(items, issueMeta, (group) => ({
title: "Homelab Manager — Alert settled",
message: `${group.length === 1 ? "An alert that was flapping has" : `${group.length} alerts that were flapping have`} been steady for an hour:\n\n${group.map((i) => `${i.active ? "⚠ still active" : "✓ cleared"} — ${i.message}`).join("\n")}`,
}));
}
export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
issues = await withoutSnoozed(issues);
if (issues.length === 0 || !(await eventEnabled("automationAlerts"))) return;
await notifyGrouped(issues, issueMeta, (group) => ({
title: "Homelab Manager — Automation Failed",
message: `${group.length} run${group.length !== 1 ? "s" : ""} failed:\n\n${group.map((i) => `✕ ${i.message}`).join("\n")}`,
}), (i) => i.key);
}
export async function notifyAutomationRecovered(resolved: { message: string; key?: string; severity?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
resolved = await withoutSnoozed(resolved);
if (resolved.length === 0 || !(await eventEnabled("automationAlerts"))) return;
await notifyGrouped(resolved, issueMeta, (group) => ({
title: "Homelab Manager — Automation Recovered",
message: `${group.length} no longer failing:\n\n${group.map((i) => `✓ ${i.message}`).join("\n")}`,
}));
}
export async function notifyDomainExpiry(
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number; expiresAt: string; registrar: string | null }[],
staleChecks: { name: string; error: string; lastOkAt: string | null }[] = [],
): Promise<void> {
if (expiring.length === 0 && staleChecks.length === 0) return;
if (!(await eventEnabled("domainExpiryCheck"))) return;
const sections: string[] = [];
if (expiring.length > 0) {
const lines = expiring.map((d) => {
const registrar = d.registrar ? ` [${d.registrar}]` : "";
return d.status === "expired"
? `✕ EXPIRED ${d.expiresAt} — ${d.name}${registrar}`
: `⚠ ${d.daysLeft}d left (${d.expiresAt}) — ${d.name}${registrar}`;
});
sections.push(`${expiring.length} domain${expiring.length !== 1 ? "s" : ""} need renewing:\n\n${lines.join("\n")}`);
}
if (staleChecks.length > 0) {
const lines = staleChecks.map((s) => `⚠ ${s.name} — ${s.error}${s.lastOkAt ? ` (last read ${s.lastOkAt.slice(0, 10)})` : " (never read)"}`);
sections.push(
`Couldn't refresh the registration for ${staleChecks.length} domain${staleChecks.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
);
}
await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"), { category: "expiry", severity: expiring.some((d) => d.status === "expired") ? "critical" : "warning" });
}
export async function notifyTailscaleKeyExpiry(
expiring: { integrationName: string; deviceLabel: string; daysLeft: number }[],
): Promise<void> {
if (expiring.length === 0) return;
if (!(await eventEnabled("tailscaleKeyCheck"))) return;
const lines = expiring.map(
(d) => `${d.daysLeft < 0 ? "✕ EXPIRED" : `⚠ ${d.daysLeft}d left`} — ${d.deviceLabel} [${d.integrationName}]`,
);
await notify(
"Homelab Manager — Tailscale Key Expiry",
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
{ category: "expiry", severity: expiring.some((d) => d.daysLeft < 0) ? "critical" : "warning" },
);
}
// Diagnostic-log "source" values (see integrations/*/adapter.ts and dns/adapters/*.ts) to display names.
const SOURCE_LABELS: Record<string, string> = {
cloudflare: "Cloudflare",
loopia: "Loopia",
pihole: "Pi-hole",
azure: "Azure DNS",
cpanel: "cPanel",
technitium: "Technitium",
tailscale: "Tailscale",
proxmox: "Proxmox",
synology: "Synology",
semaphore: "Semaphore",
gitea: "Gitea",
dockhand: "Dockhand",
uptimekuma: "Uptime Kuma",
phpipam: "phpIPAM",
pbs: "Proxmox Backup Server",
osticket: "osTicket",
unifi: "UniFi",
npm: "Nginx Proxy Manager",
homeassistant: "Home Assistant",
};
export function sourceLabel(source: string): string {
return SOURCE_LABELS[source] ?? source;
}
export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise<void> {
emitEvent("alert.opened", { alert: { key: `integration:${source}`, source, severity: "warning", message: `${sourceLabel(source)} has failed its last ${consecutiveFailures} calls in a row.` } });
if (!(await eventEnabled("integrationFailureAlerts"))) return;
await notify(
"Homelab Manager — Integration Down",
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
{ category: "integration", severity: "warning" },
);
}
export async function notifyIntegrationRecovered(source: string): Promise<void> {
emitEvent("alert.resolved", { alert: { key: `integration:${source}`, message: `${sourceLabel(source)} is answering again.` } });
if (!(await eventEnabled("integrationFailureAlerts"))) return;
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`, { category: "integration", severity: "warning" });
}