Files
Homelab-manager/server/src/services/notify.ts
T
bobbanandClaude Sonnet 5 ca61f2a915 Surface Proxmox VMs/LXCs with no backup coverage at all
A failing backup run is visible now, but a guest with no backup job
covering it in the first place was still a silent gap. Rather than
depending on Proxmox's /cluster/backup-info/not-backed-up-guests
endpoint (only exists on newer PVE versions), this derives coverage
from data already fetched: a guest counts as covered if any enabled
job either lists its vmid directly, or backs up "all guests" (scoped
to the job's node, if it has one) without excluding it.

Adds a warning banner plus a full table to the Proxmox page's Backups
card, and extends the existing daily "Proxmox backup failed"
notification (relabeled to mention this too) to also list uncovered
guests, gated by the same toggle.

Verified the coverage logic directly (it's a pure function, so no
fake server needed) across 7 cases: no jobs at all, an all-guests job
with an exclude list, a specific-vmids job, a node-scoped job that
shouldn't cover a guest on a different node, a disabled job providing
no real coverage, two jobs whose combined scope covers everything
neither would alone, and a realistic mixed scenario — all passed.
Confirmed the real dev database's mtime was untouched throughout.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-22 21:01:48 +02:00

318 lines
13 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import nodemailer from "nodemailer";
import { getSettings, type NotificationEvents } from "./settingsStore.js";
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
async function sendGotify(title: string, message: string) {
const { gotify } = await getSettings();
if (!gotify.enabled || !gotify.url || !gotify.token) return;
const base = gotify.url.replace(/\/$/, "");
try {
const res = await fetch(`${base}/message?token=${encodeURIComponent(gotify.token)}`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ title, message, priority: gotify.priority ?? 5 }),
});
if (!res.ok) console.error(`[notify] Gotify error ${res.status}: ${await res.text().catch(() => "")}`);
} catch (err) {
console.error("[notify] Gotify failed:", err instanceof Error ? err.message : err);
}
}
async function sendNtfy(title: string, message: string) {
const { ntfy } = await getSettings();
if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return;
const base = ntfy.url.replace(/\/$/, "");
const headers: Record<string, string> = {
"Content-Type": "text/plain",
Title: title,
Priority: String(ntfy.priority ?? 3),
};
if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`;
try {
const res = await fetch(`${base}/${encodeURIComponent(ntfy.topic)}`, { method: "POST", headers, body: message });
if (!res.ok) console.error(`[notify] ntfy error ${res.status}: ${await res.text().catch(() => "")}`);
} catch (err) {
console.error("[notify] ntfy failed:", err instanceof Error ? err.message : err);
}
}
async function sendSmtp(title: string, message: string) {
const { smtp } = await getSettings();
if (!smtp.enabled || !smtp.host || !smtp.to || !smtp.from) return;
try {
const transporter = nodemailer.createTransport({
host: smtp.host,
port: Number(smtp.port) || 587,
secure: !!smtp.secure,
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
});
await transporter.sendMail({ from: smtp.from, to: smtp.to, subject: title, text: message });
} catch (err) {
console.error("[notify] SMTP failed:", err instanceof Error ? err.message : err);
}
}
async function sendWebhook(title: string, message: string) {
const { webhook } = await getSettings();
if (!webhook.enabled || !webhook.url) return;
const headers: Record<string, string> = { "Content-Type": "application/json" };
if (webhook.secret) headers["X-Webhook-Secret"] = webhook.secret;
try {
const res = await fetch(webhook.url, {
method: "POST",
headers,
body: JSON.stringify({ content: `**${title}**\n${message}` }),
});
if (!res.ok) console.error(`[notify] Webhook error ${res.status}: ${await res.text().catch(() => "")}`);
} catch (err) {
console.error("[notify] Webhook failed:", err instanceof Error ? err.message : err);
}
}
async function sendToChannels(title: string, message: string): Promise<void> {
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message)]);
}
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
const parts = new Intl.DateTimeFormat("en-GB", { timeZone: timezone, hour: "2-digit", minute: "2-digit", hour12: false }).formatToParts(
new Date(),
);
return {
hh: Number(parts.find((p) => p.type === "hour")?.value ?? "0"),
mm: Number(parts.find((p) => p.type === "minute")?.value ?? "0"),
};
}
function toMinutes(hhmm: string): number {
const [h, m] = hhmm.split(":").map(Number);
return (Number.isFinite(h) ? h : 0) * 60 + (Number.isFinite(m) ? m : 0);
}
async function isQuietHoursActive(): Promise<boolean> {
const { quietHours, notifications } = await getSettings();
if (!quietHours.enabled) return false;
const { hh, mm } = currentTimeInZone(notifications.timezone);
const nowMin = hh * 60 + mm;
const startMin = toMinutes(quietHours.start);
const endMin = toMinutes(quietHours.end);
if (startMin === endMin) return false; // zero-length window — treat as always off
// A same-day window (e.g. 09:00 -> 17:00) is a simple range; a window that
// wraps past midnight (e.g. 22:00 -> 07:00, the common case) means "active"
// is everything outside [end, start) instead.
return startMin < endMin ? nowMin >= startMin && nowMin < endMin : nowMin >= startMin || nowMin < endMin;
}
export async function notify(title: string, message: string): Promise<void> {
if (await isQuietHoursActive()) {
await enqueueNotification(title, message);
return;
}
await sendToChannels(title, message);
}
/** Called by the daily scheduler at quiet-hours end (or manually) — sends everything queued during the window as one digest, then clears it. */
export async function flushQuietHoursQueue(): Promise<number> {
const queued = await listQueuedNotifications();
if (queued.length === 0) return 0;
const sections = queued.map((q) => `— ${q.title} —\n${q.message}`);
await sendToChannels(
"Homelab Manager — Notifications from quiet hours",
`${queued.length} notification${queued.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
);
await clearQueuedNotifications();
return queued.length;
}
// ─── Test senders — throw on failure so the route can report it ────────────
export async function testGotify(cfg: { url: string; token: string; priority?: number }): Promise<void> {
const base = cfg.url.replace(/\/$/, "");
const res = await fetch(`${base}/message?token=${encodeURIComponent(cfg.token)}`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
title: "Homelab Manager — Test",
message: "Gotify notifications are working correctly.",
priority: cfg.priority ?? 5,
}),
});
if (!res.ok) throw new Error(`Gotify returned ${res.status}: ${await res.text().catch(() => "")}`);
}
export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise<void> {
const base = cfg.url.replace(/\/$/, "");
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: "Homelab Manager — Test", Priority: String(cfg.priority ?? 3) };
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, {
method: "POST",
headers,
body: "ntfy notifications are working correctly.",
});
if (!res.ok) throw new Error(`ntfy returned ${res.status}: ${await res.text().catch(() => "")}`);
}
export async function testSmtp(cfg: {
host: string;
port?: number;
secure?: boolean;
username?: string;
password?: string;
from: string;
to: string;
}): Promise<void> {
const transporter = nodemailer.createTransport({
host: cfg.host,
port: Number(cfg.port) || 587,
secure: !!cfg.secure,
auth: cfg.username ? { user: cfg.username, pass: cfg.password } : undefined,
});
await transporter.verify();
await transporter.sendMail({ from: cfg.from, to: cfg.to, subject: "Homelab Manager — Test", text: "SMTP notifications are working correctly." });
}
export async function testWebhook(cfg: { url: string; secret?: string }): Promise<void> {
const headers: Record<string, string> = { "Content-Type": "application/json" };
if (cfg.secret) headers["X-Webhook-Secret"] = cfg.secret;
const res = await fetch(cfg.url, {
method: "POST",
headers,
body: JSON.stringify({ content: "**Homelab Manager — Test**\nWebhook notifications are working correctly." }),
});
if (!res.ok) throw new Error(`Webhook returned ${res.status}: ${await res.text().catch(() => "")}`);
}
// ─── Event helpers ───────────────────────────────────────────────────────────
async function eventEnabled(key: keyof NotificationEvents): Promise<boolean> {
const { notifications } = await getSettings();
return notifications[key] !== false;
}
export async function notifyDnsRecordAdded(
providerLabel: string,
zoneName: string,
record: { type: string; name: string; content: string },
): Promise<void> {
if (!(await eventEnabled("dnsAdd"))) return;
await notify("DNS Record Added", `[${providerLabel}] ${zoneName}\n+ ${record.type} ${record.name} → ${record.content}`);
}
export async function notifyDnsRecordUpdated(
providerLabel: string,
zoneName: string,
record: { type: string; name: string; content: string },
): Promise<void> {
if (!(await eventEnabled("dnsUpdate"))) return;
await notify("DNS Record Updated", `[${providerLabel}] ${zoneName}\n✎ ${record.type} ${record.name} → ${record.content}`);
}
export async function notifyDnsRecordDeleted(
providerLabel: string,
zoneName: string,
record: { type: string; name: string; content: string },
): Promise<void> {
if (!(await eventEnabled("dnsDelete"))) return;
await notify("DNS Record Deleted", `[${providerLabel}] ${zoneName}\n− ${record.type} ${record.name} ${record.content}`);
}
export async function notifySecretExpiry(
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number }[],
): Promise<void> {
if (expiring.length === 0) return;
if (!(await eventEnabled("secretCheck"))) return;
const lines = expiring.map((s) => (s.status === "expired" ? `✕ EXPIRED — ${s.name}` : `⚠ ${s.daysLeft}d left — ${s.name}`));
await notify(
"Homelab Manager — Secrets Alert",
`${expiring.length} secret${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
);
}
export async function notifyDockerUpdates(
updatesAvailable: { integrationName: string; containerName: string; environmentName: string; newerVersion: string | null }[],
): Promise<void> {
if (updatesAvailable.length === 0) return;
if (!(await eventEnabled("dockerUpdateCheck"))) return;
const lines = updatesAvailable.map(
(u) => `${u.containerName} [${u.environmentName}, ${u.integrationName}]${u.newerVersion ? ` → ${u.newerVersion}` : ""}`,
);
await notify(
"Homelab Manager — Docker Updates Available",
`${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`,
);
}
export async function notifyProxmoxBackupFailure(
failures: { integrationName: string; node: string; guestId: string | null; status: string }[],
): Promise<void> {
if (failures.length === 0) return;
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
const lines = failures.map(
(f) => `${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} [${f.integrationName}]: ${f.status}`,
);
await notify(
"Homelab Manager — Proxmox Backup Failed",
`${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
);
}
export async function notifyProxmoxUncoveredGuests(
uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[],
): Promise<void> {
if (uncovered.length === 0) return;
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
const lines = uncovered.map((u) => `${u.guestName} (#${u.vmid}) on ${u.node} [${u.integrationName}]`);
await notify(
"Homelab Manager — Proxmox Guests Without Backup Coverage",
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
);
}
export async function notifyTailscaleKeyExpiry(
expiring: { integrationName: string; deviceLabel: string; daysLeft: number }[],
): Promise<void> {
if (expiring.length === 0) return;
if (!(await eventEnabled("tailscaleKeyCheck"))) return;
const lines = expiring.map(
(d) => `${d.daysLeft < 0 ? "✕ EXPIRED" : `⚠ ${d.daysLeft}d left`} — ${d.deviceLabel} [${d.integrationName}]`,
);
await notify(
"Homelab Manager — Tailscale Key Expiry",
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
);
}
// Diagnostic-log "source" values (see integrations/*/adapter.ts and dns/adapters/*.ts) to display names.
const SOURCE_LABELS: Record<string, string> = {
cloudflare: "Cloudflare",
loopia: "Loopia",
pihole: "Pi-hole",
azure: "Azure DNS",
cpanel: "cPanel",
technitium: "Technitium",
tailscale: "Tailscale",
proxmox: "Proxmox",
synology: "Synology",
semaphore: "Semaphore",
gitea: "Gitea",
dockhand: "Dockhand",
};
function sourceLabel(source: string): string {
return SOURCE_LABELS[source] ?? source;
}
export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise<void> {
if (!(await eventEnabled("integrationFailureAlerts"))) return;
await notify(
"Homelab Manager — Integration Down",
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
);
}
export async function notifyIntegrationRecovered(source: string): Promise<void> {
if (!(await eventEnabled("integrationFailureAlerts"))) return;
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
}