Files
Homelab-manager/server/src/services/notify.ts
T
bobbanandClaude Sonnet 5.5 ec9c2f73f0 Dependencies beyond servers, web push, inventory auto-fill with QR labels, Windows agent health
- Dependencies: servers and integrations can depend on each other; alerts are folded under the root cause, and a failure explained by something down isn't announced
- Web push: hand-written VAPID and RFC 8291 encryption, per-device subscriptions, a fifth notification channel, service worker handlers
- Inventory: maker/model/serial from the agents fill blank fields, "Refresh from agent", QR label printing and ?asset= deep links
- Windows agent (v4) reports failed services, disk health, Storage Spaces pools and hardware identity; Linux agent (v5) reports hardware identity

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-18 18:53:30 +02:00

461 lines
21 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import nodemailer from "nodemailer";
import { getSettings, type NotificationEvents } from "./settingsStore.js";
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
import { emitEvent } from "./eventWebhooks.js";
import { sendPush, subscriptionCount } from "./webPush.js";
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
async function sendGotify(title: string, message: string) {
const { gotify } = await getSettings();
if (!gotify.enabled || !gotify.url || !gotify.token) return;
const base = gotify.url.replace(/\/$/, "");
try {
const res = await fetch(`${base}/message?token=${encodeURIComponent(gotify.token)}`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ title, message, priority: gotify.priority ?? 5 }),
});
if (!res.ok) console.error(`[notify] Gotify error ${res.status}: ${await res.text().catch(() => "")}`);
} catch (err) {
console.error("[notify] Gotify failed:", err instanceof Error ? err.message : err);
}
}
async function sendNtfy(title: string, message: string) {
const { ntfy } = await getSettings();
if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return;
const base = ntfy.url.replace(/\/$/, "");
const headers: Record<string, string> = {
"Content-Type": "text/plain",
Title: title,
Priority: String(ntfy.priority ?? 3),
};
if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`;
try {
const res = await fetch(`${base}/${encodeURIComponent(ntfy.topic)}`, { method: "POST", headers, body: message });
if (!res.ok) console.error(`[notify] ntfy error ${res.status}: ${await res.text().catch(() => "")}`);
} catch (err) {
console.error("[notify] ntfy failed:", err instanceof Error ? err.message : err);
}
}
async function sendSmtp(title: string, message: string) {
const { smtp } = await getSettings();
if (!smtp.enabled || !smtp.host || !smtp.to || !smtp.from) return;
try {
const transporter = nodemailer.createTransport({
host: smtp.host,
port: Number(smtp.port) || 587,
secure: !!smtp.secure,
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
});
await transporter.sendMail({ from: smtp.from, to: smtp.to, subject: title, text: message });
} catch (err) {
console.error("[notify] SMTP failed:", err instanceof Error ? err.message : err);
}
}
async function sendWebhook(title: string, message: string) {
const { webhook } = await getSettings();
if (!webhook.enabled || !webhook.url) return;
const headers: Record<string, string> = { "Content-Type": "application/json" };
if (webhook.secret) headers["X-Webhook-Secret"] = webhook.secret;
try {
const res = await fetch(webhook.url, {
method: "POST",
headers,
body: JSON.stringify({ content: `**${title}**\n${message}` }),
});
if (!res.ok) console.error(`[notify] Webhook error ${res.status}: ${await res.text().catch(() => "")}`);
} catch (err) {
console.error("[notify] Webhook failed:", err instanceof Error ? err.message : err);
}
}
async function sendWebPush(title: string, message: string) {
const { webPush } = await getSettings();
if (!webPush.enabled) return;
try {
await sendPush({ title, body: message });
} catch (err) {
console.error("[notify] Web push failed:", err instanceof Error ? err.message : err);
}
}
async function sendToChannels(title: string, message: string): Promise<void> {
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message), sendWebPush(title, message)]);
}
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
const parts = new Intl.DateTimeFormat("en-GB", { timeZone: timezone, hour: "2-digit", minute: "2-digit", hour12: false }).formatToParts(
new Date(),
);
return {
hh: Number(parts.find((p) => p.type === "hour")?.value ?? "0"),
mm: Number(parts.find((p) => p.type === "minute")?.value ?? "0"),
};
}
function toMinutes(hhmm: string): number {
const [h, m] = hhmm.split(":").map(Number);
return (Number.isFinite(h) ? h : 0) * 60 + (Number.isFinite(m) ? m : 0);
}
async function isQuietHoursActive(): Promise<boolean> {
const { quietHours, notifications } = await getSettings();
if (!quietHours.enabled) return false;
const { hh, mm } = currentTimeInZone(notifications.timezone);
const nowMin = hh * 60 + mm;
const startMin = toMinutes(quietHours.start);
const endMin = toMinutes(quietHours.end);
if (startMin === endMin) return false; // zero-length window — treat as always off
// A same-day window (e.g. 09:00 -> 17:00) is a simple range; a window that
// wraps past midnight (e.g. 22:00 -> 07:00, the common case) means "active"
// is everything outside [end, start) instead.
return startMin < endMin ? nowMin >= startMin && nowMin < endMin : nowMin >= startMin || nowMin < endMin;
}
/** Sends at once through every channel that is switched on, whatever the time — for something a person just asked for. */
export async function notifyNow(title: string, message: string): Promise<void> {
await sendToChannels(title, message);
}
/** How many channels are switched on and filled in enough to send, so a "send it now" can say when there is nowhere to send to. */
export async function enabledChannelCount(): Promise<number> {
const { gotify, ntfy, smtp, webhook, webPush } = await getSettings();
return [
webPush.enabled && (await subscriptionCount()) > 0,
gotify.enabled && gotify.url && gotify.token,
ntfy.enabled && ntfy.url && ntfy.topic,
smtp.enabled && smtp.host && smtp.to && smtp.from,
webhook.enabled && webhook.url,
].filter(Boolean).length;
}
export async function notify(title: string, message: string): Promise<void> {
if (await isQuietHoursActive()) {
await enqueueNotification(title, message);
return;
}
await sendToChannels(title, message);
}
/** Called by the daily scheduler at quiet-hours end (or manually) — sends everything queued during the window as one digest, then clears it. */
export async function flushQuietHoursQueue(): Promise<number> {
const queued = await listQueuedNotifications();
if (queued.length === 0) return 0;
const sections = queued.map((q) => `— ${q.title} —\n${q.message}`);
await sendToChannels(
"Homelab Manager — Notifications from quiet hours",
`${queued.length} notification${queued.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
);
await clearQueuedNotifications();
return queued.length;
}
// ─── Test senders — throw on failure so the route can report it ────────────
export async function testGotify(cfg: { url: string; token: string; priority?: number }): Promise<void> {
const base = cfg.url.replace(/\/$/, "");
const res = await fetch(`${base}/message?token=${encodeURIComponent(cfg.token)}`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
title: "Homelab Manager — Test",
message: "Gotify notifications are working correctly.",
priority: cfg.priority ?? 5,
}),
});
if (!res.ok) throw new Error(`Gotify returned ${res.status}: ${await res.text().catch(() => "")}`);
}
export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise<void> {
const base = cfg.url.replace(/\/$/, "");
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: "Homelab Manager — Test", Priority: String(cfg.priority ?? 3) };
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, {
method: "POST",
headers,
body: "ntfy notifications are working correctly.",
});
if (!res.ok) throw new Error(`ntfy returned ${res.status}: ${await res.text().catch(() => "")}`);
}
export async function testSmtp(cfg: {
host: string;
port?: number;
secure?: boolean;
username?: string;
password?: string;
from: string;
to: string;
}): Promise<void> {
const transporter = nodemailer.createTransport({
host: cfg.host,
port: Number(cfg.port) || 587,
secure: !!cfg.secure,
auth: cfg.username ? { user: cfg.username, pass: cfg.password } : undefined,
});
await transporter.verify();
await transporter.sendMail({ from: cfg.from, to: cfg.to, subject: "Homelab Manager — Test", text: "SMTP notifications are working correctly." });
}
export async function testWebhook(cfg: { url: string; secret?: string }): Promise<void> {
const headers: Record<string, string> = { "Content-Type": "application/json" };
if (cfg.secret) headers["X-Webhook-Secret"] = cfg.secret;
const res = await fetch(cfg.url, {
method: "POST",
headers,
body: JSON.stringify({ content: "**Homelab Manager — Test**\nWebhook notifications are working correctly." }),
});
if (!res.ok) throw new Error(`Webhook returned ${res.status}: ${await res.text().catch(() => "")}`);
}
// ─── Event helpers ───────────────────────────────────────────────────────────
async function eventEnabled(key: keyof NotificationEvents): Promise<boolean> {
const { notifications } = await getSettings();
return notifications[key] !== false;
}
export async function notifyDnsRecordAdded(
providerLabel: string,
zoneName: string,
record: { type: string; name: string; content: string },
): Promise<void> {
if (!(await eventEnabled("dnsAdd"))) return;
await notify("DNS Record Added", `[${providerLabel}] ${zoneName}\n+ ${record.type} ${record.name} → ${record.content}`);
}
export async function notifyDnsRecordUpdated(
providerLabel: string,
zoneName: string,
record: { type: string; name: string; content: string },
): Promise<void> {
if (!(await eventEnabled("dnsUpdate"))) return;
await notify("DNS Record Updated", `[${providerLabel}] ${zoneName}\n✎ ${record.type} ${record.name} → ${record.content}`);
}
export async function notifyDnsRecordDeleted(
providerLabel: string,
zoneName: string,
record: { type: string; name: string; content: string },
): Promise<void> {
if (!(await eventEnabled("dnsDelete"))) return;
await notify("DNS Record Deleted", `[${providerLabel}] ${zoneName}\n− ${record.type} ${record.name} ${record.content}`);
}
export async function notifySecretExpiry(
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number }[],
checkFailures: { name: string; host: string; port: number; error?: string }[] = [],
): Promise<void> {
if (expiring.length === 0 && checkFailures.length === 0) return;
if (!(await eventEnabled("secretCheck"))) return;
const sections: string[] = [];
if (expiring.length > 0) {
const lines = expiring.map((s) => (s.status === "expired" ? `✕ EXPIRED — ${s.name}` : `⚠ ${s.daysLeft}d left — ${s.name}`));
sections.push(`${expiring.length} secret${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`);
}
if (checkFailures.length > 0) {
const lines = checkFailures.map((f) => `⚠ ${f.name} (${f.host}:${f.port}) — ${f.error ?? "check failed"}`);
sections.push(
`Couldn't read the live certificate for ${checkFailures.length} monitored secret${checkFailures.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
);
}
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"));
}
export async function notifyDockerUpdates(
updatesAvailable: { integrationName: string; containerName: string; environmentName: string; newerVersion: string | null }[],
): Promise<void> {
if (updatesAvailable.length === 0) return;
if (!(await eventEnabled("dockerUpdateCheck"))) return;
const lines = updatesAvailable.map(
(u) => `${u.containerName} [${u.environmentName}, ${u.integrationName}]${u.newerVersion ? ` → ${u.newerVersion}` : ""}`,
);
await notify(
"Homelab Manager — Docker Updates Available",
`${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`,
);
}
export async function notifyProxmoxBackupFailure(
failures: { integrationName: string; node: string; guestId: string | null; status: string }[],
): Promise<void> {
if (failures.length === 0) return;
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
const lines = failures.map(
(f) => `${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} [${f.integrationName}]: ${f.status}`,
);
await notify(
"Homelab Manager — Proxmox Backup Failed",
`${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
);
}
export async function notifyProxmoxUncoveredGuests(
uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[],
): Promise<void> {
if (uncovered.length === 0) return;
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
const lines = uncovered.map((u) => `${u.guestName} (#${u.vmid}) on ${u.node} [${u.integrationName}]`);
await notify(
"Homelab Manager — Proxmox Guests Without Backup Coverage",
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
);
}
export async function notifyStaleBackups(stale: { message: string }[]): Promise<void> {
if (stale.length === 0) return;
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
await notify(
"Homelab Manager — Backups Gone Stale",
`${stale.length} guest${stale.length !== 1 ? "s have" : " has"} no recent backup:\n\n${stale.map((s) => s.message).join("\n")}`,
);
}
export async function notifyUpdateRunFinished(run: { serverName: string; templateName: string; ok: boolean; message: string }): Promise<void> {
if (!(await eventEnabled("updateRunResults"))) return;
await notify(
run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`,
`"${run.templateName}" on ${run.serverName}: ${run.message}`,
);
}
export async function notifyPbsVerificationFailed(
failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[],
): Promise<void> {
if (failures.length === 0) return;
if (!(await eventEnabled("pbsVerificationCheck"))) return;
const lines = failures.map((f) =>
f.error
? `${f.datastore} [${f.integrationName}]: couldn't be read — ${f.error}`
: `${f.datastore} [${f.integrationName}]: ${f.failedCount} snapshot${f.failedCount !== 1 ? "s" : ""} failed verification`,
);
await notify(
"Homelab Manager — Proxmox Backup Server Verification Failed",
`${failures.length} datastore${failures.length !== 1 ? "s have" : " has"} a problem:\n\n${lines.join("\n")}`,
);
}
export async function notifyHealthIssues(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
if (!(await eventEnabled("healthAlerts"))) return;
await notify(
"Homelab Manager — Health Alert",
`${issues.length} new issue${issues.length !== 1 ? "s" : ""}:\n\n${issues.map((i) => `⚠ ${i.message}`).join("\n")}`,
);
}
export async function notifyHealthRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
if (!(await eventEnabled("healthAlerts"))) return;
await notify(
"Homelab Manager — Health Recovered",
`${resolved.length} issue${resolved.length !== 1 ? "s" : ""} cleared:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
);
}
export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
if (!(await eventEnabled("automationAlerts"))) return;
await notify(
"Homelab Manager — Automation Failed",
`${issues.length} run${issues.length !== 1 ? "s" : ""} failed:\n\n${issues.map((i) => `✕ ${i.message}`).join("\n")}`,
);
}
export async function notifyAutomationRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
if (!(await eventEnabled("automationAlerts"))) return;
await notify(
"Homelab Manager — Automation Recovered",
`${resolved.length} no longer failing:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
);
}
export async function notifyDomainExpiry(
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number; expiresAt: string; registrar: string | null }[],
staleChecks: { name: string; error: string; lastOkAt: string | null }[] = [],
): Promise<void> {
if (expiring.length === 0 && staleChecks.length === 0) return;
if (!(await eventEnabled("domainExpiryCheck"))) return;
const sections: string[] = [];
if (expiring.length > 0) {
const lines = expiring.map((d) => {
const registrar = d.registrar ? ` [${d.registrar}]` : "";
return d.status === "expired"
? `✕ EXPIRED ${d.expiresAt} — ${d.name}${registrar}`
: `⚠ ${d.daysLeft}d left (${d.expiresAt}) — ${d.name}${registrar}`;
});
sections.push(`${expiring.length} domain${expiring.length !== 1 ? "s" : ""} need renewing:\n\n${lines.join("\n")}`);
}
if (staleChecks.length > 0) {
const lines = staleChecks.map((s) => `⚠ ${s.name} — ${s.error}${s.lastOkAt ? ` (last read ${s.lastOkAt.slice(0, 10)})` : " (never read)"}`);
sections.push(
`Couldn't refresh the registration for ${staleChecks.length} domain${staleChecks.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
);
}
await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"));
}
export async function notifyTailscaleKeyExpiry(
expiring: { integrationName: string; deviceLabel: string; daysLeft: number }[],
): Promise<void> {
if (expiring.length === 0) return;
if (!(await eventEnabled("tailscaleKeyCheck"))) return;
const lines = expiring.map(
(d) => `${d.daysLeft < 0 ? "✕ EXPIRED" : `⚠ ${d.daysLeft}d left`} — ${d.deviceLabel} [${d.integrationName}]`,
);
await notify(
"Homelab Manager — Tailscale Key Expiry",
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
);
}
// Diagnostic-log "source" values (see integrations/*/adapter.ts and dns/adapters/*.ts) to display names.
const SOURCE_LABELS: Record<string, string> = {
cloudflare: "Cloudflare",
loopia: "Loopia",
pihole: "Pi-hole",
azure: "Azure DNS",
cpanel: "cPanel",
technitium: "Technitium",
tailscale: "Tailscale",
proxmox: "Proxmox",
synology: "Synology",
semaphore: "Semaphore",
gitea: "Gitea",
dockhand: "Dockhand",
uptimekuma: "Uptime Kuma",
phpipam: "phpIPAM",
pbs: "Proxmox Backup Server",
osticket: "osTicket",
};
export function sourceLabel(source: string): string {
return SOURCE_LABELS[source] ?? source;
}
export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise<void> {
emitEvent("alert.opened", { alert: { key: `integration:${source}`, source, severity: "warning", message: `${sourceLabel(source)} has failed its last ${consecutiveFailures} calls in a row.` } });
if (!(await eventEnabled("integrationFailureAlerts"))) return;
await notify(
"Homelab Manager — Integration Down",
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
);
}
export async function notifyIntegrationRecovered(source: string): Promise<void> {
emitEvent("alert.resolved", { alert: { key: `integration:${source}`, message: `${sourceLabel(source)} is answering again.` } });
if (!(await eventEnabled("integrationFailureAlerts"))) return;
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
}