A failing backup run is visible now, but a guest with no backup job covering it in the first place was still a silent gap. Rather than depending on Proxmox's /cluster/backup-info/not-backed-up-guests endpoint (only exists on newer PVE versions), this derives coverage from data already fetched: a guest counts as covered if any enabled job either lists its vmid directly, or backs up "all guests" (scoped to the job's node, if it has one) without excluding it. Adds a warning banner plus a full table to the Proxmox page's Backups card, and extends the existing daily "Proxmox backup failed" notification (relabeled to mention this too) to also list uncovered guests, gated by the same toggle. Verified the coverage logic directly (it's a pure function, so no fake server needed) across 7 cases: no jobs at all, an all-guests job with an exclude list, a specific-vmids job, a node-scoped job that shouldn't cover a guest on a different node, a disabled job providing no real coverage, two jobs whose combined scope covers everything neither would alone, and a realistic mixed scenario — all passed. Confirmed the real dev database's mtime was untouched throughout. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
318 lines
13 KiB
TypeScript
318 lines
13 KiB
TypeScript
import nodemailer from "nodemailer";
|
||
import { getSettings, type NotificationEvents } from "./settingsStore.js";
|
||
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
|
||
|
||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||
|
||
async function sendGotify(title: string, message: string) {
|
||
const { gotify } = await getSettings();
|
||
if (!gotify.enabled || !gotify.url || !gotify.token) return;
|
||
const base = gotify.url.replace(/\/$/, "");
|
||
try {
|
||
const res = await fetch(`${base}/message?token=${encodeURIComponent(gotify.token)}`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({ title, message, priority: gotify.priority ?? 5 }),
|
||
});
|
||
if (!res.ok) console.error(`[notify] Gotify error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] Gotify failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendNtfy(title: string, message: string) {
|
||
const { ntfy } = await getSettings();
|
||
if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return;
|
||
const base = ntfy.url.replace(/\/$/, "");
|
||
const headers: Record<string, string> = {
|
||
"Content-Type": "text/plain",
|
||
Title: title,
|
||
Priority: String(ntfy.priority ?? 3),
|
||
};
|
||
if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`;
|
||
try {
|
||
const res = await fetch(`${base}/${encodeURIComponent(ntfy.topic)}`, { method: "POST", headers, body: message });
|
||
if (!res.ok) console.error(`[notify] ntfy error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] ntfy failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendSmtp(title: string, message: string) {
|
||
const { smtp } = await getSettings();
|
||
if (!smtp.enabled || !smtp.host || !smtp.to || !smtp.from) return;
|
||
try {
|
||
const transporter = nodemailer.createTransport({
|
||
host: smtp.host,
|
||
port: Number(smtp.port) || 587,
|
||
secure: !!smtp.secure,
|
||
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
|
||
});
|
||
await transporter.sendMail({ from: smtp.from, to: smtp.to, subject: title, text: message });
|
||
} catch (err) {
|
||
console.error("[notify] SMTP failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendWebhook(title: string, message: string) {
|
||
const { webhook } = await getSettings();
|
||
if (!webhook.enabled || !webhook.url) return;
|
||
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
||
if (webhook.secret) headers["X-Webhook-Secret"] = webhook.secret;
|
||
try {
|
||
const res = await fetch(webhook.url, {
|
||
method: "POST",
|
||
headers,
|
||
body: JSON.stringify({ content: `**${title}**\n${message}` }),
|
||
});
|
||
if (!res.ok) console.error(`[notify] Webhook error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] Webhook failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendToChannels(title: string, message: string): Promise<void> {
|
||
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message)]);
|
||
}
|
||
|
||
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
|
||
const parts = new Intl.DateTimeFormat("en-GB", { timeZone: timezone, hour: "2-digit", minute: "2-digit", hour12: false }).formatToParts(
|
||
new Date(),
|
||
);
|
||
return {
|
||
hh: Number(parts.find((p) => p.type === "hour")?.value ?? "0"),
|
||
mm: Number(parts.find((p) => p.type === "minute")?.value ?? "0"),
|
||
};
|
||
}
|
||
|
||
function toMinutes(hhmm: string): number {
|
||
const [h, m] = hhmm.split(":").map(Number);
|
||
return (Number.isFinite(h) ? h : 0) * 60 + (Number.isFinite(m) ? m : 0);
|
||
}
|
||
|
||
async function isQuietHoursActive(): Promise<boolean> {
|
||
const { quietHours, notifications } = await getSettings();
|
||
if (!quietHours.enabled) return false;
|
||
const { hh, mm } = currentTimeInZone(notifications.timezone);
|
||
const nowMin = hh * 60 + mm;
|
||
const startMin = toMinutes(quietHours.start);
|
||
const endMin = toMinutes(quietHours.end);
|
||
if (startMin === endMin) return false; // zero-length window — treat as always off
|
||
// A same-day window (e.g. 09:00 -> 17:00) is a simple range; a window that
|
||
// wraps past midnight (e.g. 22:00 -> 07:00, the common case) means "active"
|
||
// is everything outside [end, start) instead.
|
||
return startMin < endMin ? nowMin >= startMin && nowMin < endMin : nowMin >= startMin || nowMin < endMin;
|
||
}
|
||
|
||
export async function notify(title: string, message: string): Promise<void> {
|
||
if (await isQuietHoursActive()) {
|
||
await enqueueNotification(title, message);
|
||
return;
|
||
}
|
||
await sendToChannels(title, message);
|
||
}
|
||
|
||
/** Called by the daily scheduler at quiet-hours end (or manually) — sends everything queued during the window as one digest, then clears it. */
|
||
export async function flushQuietHoursQueue(): Promise<number> {
|
||
const queued = await listQueuedNotifications();
|
||
if (queued.length === 0) return 0;
|
||
const sections = queued.map((q) => `— ${q.title} —\n${q.message}`);
|
||
await sendToChannels(
|
||
"Homelab Manager — Notifications from quiet hours",
|
||
`${queued.length} notification${queued.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
|
||
);
|
||
await clearQueuedNotifications();
|
||
return queued.length;
|
||
}
|
||
|
||
// ─── Test senders — throw on failure so the route can report it ────────────
|
||
|
||
export async function testGotify(cfg: { url: string; token: string; priority?: number }): Promise<void> {
|
||
const base = cfg.url.replace(/\/$/, "");
|
||
const res = await fetch(`${base}/message?token=${encodeURIComponent(cfg.token)}`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
title: "Homelab Manager — Test",
|
||
message: "Gotify notifications are working correctly.",
|
||
priority: cfg.priority ?? 5,
|
||
}),
|
||
});
|
||
if (!res.ok) throw new Error(`Gotify returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise<void> {
|
||
const base = cfg.url.replace(/\/$/, "");
|
||
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: "Homelab Manager — Test", Priority: String(cfg.priority ?? 3) };
|
||
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
|
||
const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, {
|
||
method: "POST",
|
||
headers,
|
||
body: "ntfy notifications are working correctly.",
|
||
});
|
||
if (!res.ok) throw new Error(`ntfy returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
export async function testSmtp(cfg: {
|
||
host: string;
|
||
port?: number;
|
||
secure?: boolean;
|
||
username?: string;
|
||
password?: string;
|
||
from: string;
|
||
to: string;
|
||
}): Promise<void> {
|
||
const transporter = nodemailer.createTransport({
|
||
host: cfg.host,
|
||
port: Number(cfg.port) || 587,
|
||
secure: !!cfg.secure,
|
||
auth: cfg.username ? { user: cfg.username, pass: cfg.password } : undefined,
|
||
});
|
||
await transporter.verify();
|
||
await transporter.sendMail({ from: cfg.from, to: cfg.to, subject: "Homelab Manager — Test", text: "SMTP notifications are working correctly." });
|
||
}
|
||
|
||
export async function testWebhook(cfg: { url: string; secret?: string }): Promise<void> {
|
||
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
||
if (cfg.secret) headers["X-Webhook-Secret"] = cfg.secret;
|
||
const res = await fetch(cfg.url, {
|
||
method: "POST",
|
||
headers,
|
||
body: JSON.stringify({ content: "**Homelab Manager — Test**\nWebhook notifications are working correctly." }),
|
||
});
|
||
if (!res.ok) throw new Error(`Webhook returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
// ─── Event helpers ───────────────────────────────────────────────────────────
|
||
|
||
async function eventEnabled(key: keyof NotificationEvents): Promise<boolean> {
|
||
const { notifications } = await getSettings();
|
||
return notifications[key] !== false;
|
||
}
|
||
|
||
export async function notifyDnsRecordAdded(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsAdd"))) return;
|
||
await notify("DNS Record Added", `[${providerLabel}] ${zoneName}\n+ ${record.type} ${record.name} → ${record.content}`);
|
||
}
|
||
|
||
export async function notifyDnsRecordUpdated(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsUpdate"))) return;
|
||
await notify("DNS Record Updated", `[${providerLabel}] ${zoneName}\n✎ ${record.type} ${record.name} → ${record.content}`);
|
||
}
|
||
|
||
export async function notifyDnsRecordDeleted(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsDelete"))) return;
|
||
await notify("DNS Record Deleted", `[${providerLabel}] ${zoneName}\n− ${record.type} ${record.name} ${record.content}`);
|
||
}
|
||
|
||
export async function notifySecretExpiry(
|
||
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number }[],
|
||
): Promise<void> {
|
||
if (expiring.length === 0) return;
|
||
if (!(await eventEnabled("secretCheck"))) return;
|
||
const lines = expiring.map((s) => (s.status === "expired" ? `✕ EXPIRED — ${s.name}` : `⚠ ${s.daysLeft}d left — ${s.name}`));
|
||
await notify(
|
||
"Homelab Manager — Secrets Alert",
|
||
`${expiring.length} secret${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyDockerUpdates(
|
||
updatesAvailable: { integrationName: string; containerName: string; environmentName: string; newerVersion: string | null }[],
|
||
): Promise<void> {
|
||
if (updatesAvailable.length === 0) return;
|
||
if (!(await eventEnabled("dockerUpdateCheck"))) return;
|
||
const lines = updatesAvailable.map(
|
||
(u) => `${u.containerName} [${u.environmentName}, ${u.integrationName}]${u.newerVersion ? ` → ${u.newerVersion}` : ""}`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Docker Updates Available",
|
||
`${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyProxmoxBackupFailure(
|
||
failures: { integrationName: string; node: string; guestId: string | null; status: string }[],
|
||
): Promise<void> {
|
||
if (failures.length === 0) return;
|
||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||
const lines = failures.map(
|
||
(f) => `${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} [${f.integrationName}]: ${f.status}`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Proxmox Backup Failed",
|
||
`${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyProxmoxUncoveredGuests(
|
||
uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[],
|
||
): Promise<void> {
|
||
if (uncovered.length === 0) return;
|
||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||
const lines = uncovered.map((u) => `${u.guestName} (#${u.vmid}) on ${u.node} [${u.integrationName}]`);
|
||
await notify(
|
||
"Homelab Manager — Proxmox Guests Without Backup Coverage",
|
||
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyTailscaleKeyExpiry(
|
||
expiring: { integrationName: string; deviceLabel: string; daysLeft: number }[],
|
||
): Promise<void> {
|
||
if (expiring.length === 0) return;
|
||
if (!(await eventEnabled("tailscaleKeyCheck"))) return;
|
||
const lines = expiring.map(
|
||
(d) => `${d.daysLeft < 0 ? "✕ EXPIRED" : `⚠ ${d.daysLeft}d left`} — ${d.deviceLabel} [${d.integrationName}]`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Tailscale Key Expiry",
|
||
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
// Diagnostic-log "source" values (see integrations/*/adapter.ts and dns/adapters/*.ts) to display names.
|
||
const SOURCE_LABELS: Record<string, string> = {
|
||
cloudflare: "Cloudflare",
|
||
loopia: "Loopia",
|
||
pihole: "Pi-hole",
|
||
azure: "Azure DNS",
|
||
cpanel: "cPanel",
|
||
technitium: "Technitium",
|
||
tailscale: "Tailscale",
|
||
proxmox: "Proxmox",
|
||
synology: "Synology",
|
||
semaphore: "Semaphore",
|
||
gitea: "Gitea",
|
||
dockhand: "Dockhand",
|
||
};
|
||
|
||
function sourceLabel(source: string): string {
|
||
return SOURCE_LABELS[source] ?? source;
|
||
}
|
||
|
||
export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise<void> {
|
||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Integration Down",
|
||
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
|
||
);
|
||
}
|
||
|
||
export async function notifyIntegrationRecovered(source: string): Promise<void> {
|
||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
|
||
}
|