One list of the current problems across servers and integrations, instead of waiting for a notification or visiting each page: servers that stopped reporting, full or nearly full disks and volumes (critical from 95%), Synology volume/disk problems, failed or uncovered Proxmox backups, failed Proxmox Backup Server verifications, container image updates, expired or expiring secrets/domains/Tailscale keys, failed Semaphore and Gitea runs, Uptime Kuma monitors that are down, overdue osTicket tickets, and integrations whose calls keep failing. Visible to every role, with severity and kind filters, search, sorting, CSV export and "Check now". It runs the same checks that send the notifications rather than a second copy of them: the detection in the health, automation, Proxmox backup, PBS, Docker update and Tailscale key checks is pulled out into shared collectors that both the schedulers and the page call, so the two can't disagree about what counts as a problem. Notification behaviour is unchanged, including the scheduled backup checks skipping integrations under a maintenance window. Unlike the notifications the page ignores the on/off toggles, and keeps problems under a maintenance window, marked silenced and counted apart. It reads live, so a result is reused for a minute (and Refresh can't re-run everything more than once every ten seconds), and every source has a 20 s limit so one hung integration can't hang the page. Anything it couldn't read is called out at the top instead of looking like all clear, and server checks pause for the same 20 minutes after a restart as the notifications do, with a note saying so. Also gives the newer integrations (PBS, osTicket, Uptime Kuma, phpIPAM) proper names in "integration down" notifications instead of their ids. Verified through the real routes against a scratch database with fake backends (offline and full-disk servers, secrets and domains, a silenced server, a fake PBS with failed verification, a hanging integration, a refused one, a failing-calls streak, caching, the restart grace period, auth), and by rendering the real page against that data in a browser: filters, search, silenced toggle, sorting, Check now, dark mode. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
407 lines
17 KiB
TypeScript
407 lines
17 KiB
TypeScript
import nodemailer from "nodemailer";
|
||
import { getSettings, type NotificationEvents } from "./settingsStore.js";
|
||
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
|
||
|
||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||
|
||
async function sendGotify(title: string, message: string) {
|
||
const { gotify } = await getSettings();
|
||
if (!gotify.enabled || !gotify.url || !gotify.token) return;
|
||
const base = gotify.url.replace(/\/$/, "");
|
||
try {
|
||
const res = await fetch(`${base}/message?token=${encodeURIComponent(gotify.token)}`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({ title, message, priority: gotify.priority ?? 5 }),
|
||
});
|
||
if (!res.ok) console.error(`[notify] Gotify error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] Gotify failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendNtfy(title: string, message: string) {
|
||
const { ntfy } = await getSettings();
|
||
if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return;
|
||
const base = ntfy.url.replace(/\/$/, "");
|
||
const headers: Record<string, string> = {
|
||
"Content-Type": "text/plain",
|
||
Title: title,
|
||
Priority: String(ntfy.priority ?? 3),
|
||
};
|
||
if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`;
|
||
try {
|
||
const res = await fetch(`${base}/${encodeURIComponent(ntfy.topic)}`, { method: "POST", headers, body: message });
|
||
if (!res.ok) console.error(`[notify] ntfy error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] ntfy failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendSmtp(title: string, message: string) {
|
||
const { smtp } = await getSettings();
|
||
if (!smtp.enabled || !smtp.host || !smtp.to || !smtp.from) return;
|
||
try {
|
||
const transporter = nodemailer.createTransport({
|
||
host: smtp.host,
|
||
port: Number(smtp.port) || 587,
|
||
secure: !!smtp.secure,
|
||
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
|
||
});
|
||
await transporter.sendMail({ from: smtp.from, to: smtp.to, subject: title, text: message });
|
||
} catch (err) {
|
||
console.error("[notify] SMTP failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendWebhook(title: string, message: string) {
|
||
const { webhook } = await getSettings();
|
||
if (!webhook.enabled || !webhook.url) return;
|
||
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
||
if (webhook.secret) headers["X-Webhook-Secret"] = webhook.secret;
|
||
try {
|
||
const res = await fetch(webhook.url, {
|
||
method: "POST",
|
||
headers,
|
||
body: JSON.stringify({ content: `**${title}**\n${message}` }),
|
||
});
|
||
if (!res.ok) console.error(`[notify] Webhook error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] Webhook failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendToChannels(title: string, message: string): Promise<void> {
|
||
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message)]);
|
||
}
|
||
|
||
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
|
||
const parts = new Intl.DateTimeFormat("en-GB", { timeZone: timezone, hour: "2-digit", minute: "2-digit", hour12: false }).formatToParts(
|
||
new Date(),
|
||
);
|
||
return {
|
||
hh: Number(parts.find((p) => p.type === "hour")?.value ?? "0"),
|
||
mm: Number(parts.find((p) => p.type === "minute")?.value ?? "0"),
|
||
};
|
||
}
|
||
|
||
function toMinutes(hhmm: string): number {
|
||
const [h, m] = hhmm.split(":").map(Number);
|
||
return (Number.isFinite(h) ? h : 0) * 60 + (Number.isFinite(m) ? m : 0);
|
||
}
|
||
|
||
async function isQuietHoursActive(): Promise<boolean> {
|
||
const { quietHours, notifications } = await getSettings();
|
||
if (!quietHours.enabled) return false;
|
||
const { hh, mm } = currentTimeInZone(notifications.timezone);
|
||
const nowMin = hh * 60 + mm;
|
||
const startMin = toMinutes(quietHours.start);
|
||
const endMin = toMinutes(quietHours.end);
|
||
if (startMin === endMin) return false; // zero-length window — treat as always off
|
||
// A same-day window (e.g. 09:00 -> 17:00) is a simple range; a window that
|
||
// wraps past midnight (e.g. 22:00 -> 07:00, the common case) means "active"
|
||
// is everything outside [end, start) instead.
|
||
return startMin < endMin ? nowMin >= startMin && nowMin < endMin : nowMin >= startMin || nowMin < endMin;
|
||
}
|
||
|
||
export async function notify(title: string, message: string): Promise<void> {
|
||
if (await isQuietHoursActive()) {
|
||
await enqueueNotification(title, message);
|
||
return;
|
||
}
|
||
await sendToChannels(title, message);
|
||
}
|
||
|
||
/** Called by the daily scheduler at quiet-hours end (or manually) — sends everything queued during the window as one digest, then clears it. */
|
||
export async function flushQuietHoursQueue(): Promise<number> {
|
||
const queued = await listQueuedNotifications();
|
||
if (queued.length === 0) return 0;
|
||
const sections = queued.map((q) => `— ${q.title} —\n${q.message}`);
|
||
await sendToChannels(
|
||
"Homelab Manager — Notifications from quiet hours",
|
||
`${queued.length} notification${queued.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
|
||
);
|
||
await clearQueuedNotifications();
|
||
return queued.length;
|
||
}
|
||
|
||
// ─── Test senders — throw on failure so the route can report it ────────────
|
||
|
||
export async function testGotify(cfg: { url: string; token: string; priority?: number }): Promise<void> {
|
||
const base = cfg.url.replace(/\/$/, "");
|
||
const res = await fetch(`${base}/message?token=${encodeURIComponent(cfg.token)}`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
title: "Homelab Manager — Test",
|
||
message: "Gotify notifications are working correctly.",
|
||
priority: cfg.priority ?? 5,
|
||
}),
|
||
});
|
||
if (!res.ok) throw new Error(`Gotify returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise<void> {
|
||
const base = cfg.url.replace(/\/$/, "");
|
||
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: "Homelab Manager — Test", Priority: String(cfg.priority ?? 3) };
|
||
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
|
||
const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, {
|
||
method: "POST",
|
||
headers,
|
||
body: "ntfy notifications are working correctly.",
|
||
});
|
||
if (!res.ok) throw new Error(`ntfy returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
export async function testSmtp(cfg: {
|
||
host: string;
|
||
port?: number;
|
||
secure?: boolean;
|
||
username?: string;
|
||
password?: string;
|
||
from: string;
|
||
to: string;
|
||
}): Promise<void> {
|
||
const transporter = nodemailer.createTransport({
|
||
host: cfg.host,
|
||
port: Number(cfg.port) || 587,
|
||
secure: !!cfg.secure,
|
||
auth: cfg.username ? { user: cfg.username, pass: cfg.password } : undefined,
|
||
});
|
||
await transporter.verify();
|
||
await transporter.sendMail({ from: cfg.from, to: cfg.to, subject: "Homelab Manager — Test", text: "SMTP notifications are working correctly." });
|
||
}
|
||
|
||
export async function testWebhook(cfg: { url: string; secret?: string }): Promise<void> {
|
||
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
||
if (cfg.secret) headers["X-Webhook-Secret"] = cfg.secret;
|
||
const res = await fetch(cfg.url, {
|
||
method: "POST",
|
||
headers,
|
||
body: JSON.stringify({ content: "**Homelab Manager — Test**\nWebhook notifications are working correctly." }),
|
||
});
|
||
if (!res.ok) throw new Error(`Webhook returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
// ─── Event helpers ───────────────────────────────────────────────────────────
|
||
|
||
async function eventEnabled(key: keyof NotificationEvents): Promise<boolean> {
|
||
const { notifications } = await getSettings();
|
||
return notifications[key] !== false;
|
||
}
|
||
|
||
export async function notifyDnsRecordAdded(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsAdd"))) return;
|
||
await notify("DNS Record Added", `[${providerLabel}] ${zoneName}\n+ ${record.type} ${record.name} → ${record.content}`);
|
||
}
|
||
|
||
export async function notifyDnsRecordUpdated(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsUpdate"))) return;
|
||
await notify("DNS Record Updated", `[${providerLabel}] ${zoneName}\n✎ ${record.type} ${record.name} → ${record.content}`);
|
||
}
|
||
|
||
export async function notifyDnsRecordDeleted(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsDelete"))) return;
|
||
await notify("DNS Record Deleted", `[${providerLabel}] ${zoneName}\n− ${record.type} ${record.name} ${record.content}`);
|
||
}
|
||
|
||
export async function notifySecretExpiry(
|
||
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number }[],
|
||
checkFailures: { name: string; host: string; port: number; error?: string }[] = [],
|
||
): Promise<void> {
|
||
if (expiring.length === 0 && checkFailures.length === 0) return;
|
||
if (!(await eventEnabled("secretCheck"))) return;
|
||
const sections: string[] = [];
|
||
if (expiring.length > 0) {
|
||
const lines = expiring.map((s) => (s.status === "expired" ? `✕ EXPIRED — ${s.name}` : `⚠ ${s.daysLeft}d left — ${s.name}`));
|
||
sections.push(`${expiring.length} secret${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`);
|
||
}
|
||
if (checkFailures.length > 0) {
|
||
const lines = checkFailures.map((f) => `⚠ ${f.name} (${f.host}:${f.port}) — ${f.error ?? "check failed"}`);
|
||
sections.push(
|
||
`Couldn't read the live certificate for ${checkFailures.length} monitored secret${checkFailures.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"));
|
||
}
|
||
|
||
export async function notifyDockerUpdates(
|
||
updatesAvailable: { integrationName: string; containerName: string; environmentName: string; newerVersion: string | null }[],
|
||
): Promise<void> {
|
||
if (updatesAvailable.length === 0) return;
|
||
if (!(await eventEnabled("dockerUpdateCheck"))) return;
|
||
const lines = updatesAvailable.map(
|
||
(u) => `${u.containerName} [${u.environmentName}, ${u.integrationName}]${u.newerVersion ? ` → ${u.newerVersion}` : ""}`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Docker Updates Available",
|
||
`${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyProxmoxBackupFailure(
|
||
failures: { integrationName: string; node: string; guestId: string | null; status: string }[],
|
||
): Promise<void> {
|
||
if (failures.length === 0) return;
|
||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||
const lines = failures.map(
|
||
(f) => `${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} [${f.integrationName}]: ${f.status}`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Proxmox Backup Failed",
|
||
`${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyProxmoxUncoveredGuests(
|
||
uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[],
|
||
): Promise<void> {
|
||
if (uncovered.length === 0) return;
|
||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||
const lines = uncovered.map((u) => `${u.guestName} (#${u.vmid}) on ${u.node} [${u.integrationName}]`);
|
||
await notify(
|
||
"Homelab Manager — Proxmox Guests Without Backup Coverage",
|
||
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyPbsVerificationFailed(
|
||
failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[],
|
||
): Promise<void> {
|
||
if (failures.length === 0) return;
|
||
if (!(await eventEnabled("pbsVerificationCheck"))) return;
|
||
const lines = failures.map((f) =>
|
||
f.error
|
||
? `${f.datastore} [${f.integrationName}]: couldn't be read — ${f.error}`
|
||
: `${f.datastore} [${f.integrationName}]: ${f.failedCount} snapshot${f.failedCount !== 1 ? "s" : ""} failed verification`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Proxmox Backup Server Verification Failed",
|
||
`${failures.length} datastore${failures.length !== 1 ? "s have" : " has"} a problem:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyHealthIssues(issues: { message: string }[]): Promise<void> {
|
||
if (issues.length === 0) return;
|
||
if (!(await eventEnabled("healthAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Health Alert",
|
||
`${issues.length} new issue${issues.length !== 1 ? "s" : ""}:\n\n${issues.map((i) => `⚠ ${i.message}`).join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyHealthRecovered(resolved: { message: string }[]): Promise<void> {
|
||
if (resolved.length === 0) return;
|
||
if (!(await eventEnabled("healthAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Health Recovered",
|
||
`${resolved.length} issue${resolved.length !== 1 ? "s" : ""} cleared:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyAutomationFailed(issues: { message: string }[]): Promise<void> {
|
||
if (issues.length === 0) return;
|
||
if (!(await eventEnabled("automationAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Automation Failed",
|
||
`${issues.length} run${issues.length !== 1 ? "s" : ""} failed:\n\n${issues.map((i) => `✕ ${i.message}`).join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyAutomationRecovered(resolved: { message: string }[]): Promise<void> {
|
||
if (resolved.length === 0) return;
|
||
if (!(await eventEnabled("automationAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Automation Recovered",
|
||
`${resolved.length} no longer failing:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyDomainExpiry(
|
||
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number; expiresAt: string; registrar: string | null }[],
|
||
staleChecks: { name: string; error: string; lastOkAt: string | null }[] = [],
|
||
): Promise<void> {
|
||
if (expiring.length === 0 && staleChecks.length === 0) return;
|
||
if (!(await eventEnabled("domainExpiryCheck"))) return;
|
||
const sections: string[] = [];
|
||
if (expiring.length > 0) {
|
||
const lines = expiring.map((d) => {
|
||
const registrar = d.registrar ? ` [${d.registrar}]` : "";
|
||
return d.status === "expired"
|
||
? `✕ EXPIRED ${d.expiresAt} — ${d.name}${registrar}`
|
||
: `⚠ ${d.daysLeft}d left (${d.expiresAt}) — ${d.name}${registrar}`;
|
||
});
|
||
sections.push(`${expiring.length} domain${expiring.length !== 1 ? "s" : ""} need renewing:\n\n${lines.join("\n")}`);
|
||
}
|
||
if (staleChecks.length > 0) {
|
||
const lines = staleChecks.map((s) => `⚠ ${s.name} — ${s.error}${s.lastOkAt ? ` (last read ${s.lastOkAt.slice(0, 10)})` : " (never read)"}`);
|
||
sections.push(
|
||
`Couldn't refresh the registration for ${staleChecks.length} domain${staleChecks.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"));
|
||
}
|
||
|
||
export async function notifyTailscaleKeyExpiry(
|
||
expiring: { integrationName: string; deviceLabel: string; daysLeft: number }[],
|
||
): Promise<void> {
|
||
if (expiring.length === 0) return;
|
||
if (!(await eventEnabled("tailscaleKeyCheck"))) return;
|
||
const lines = expiring.map(
|
||
(d) => `${d.daysLeft < 0 ? "✕ EXPIRED" : `⚠ ${d.daysLeft}d left`} — ${d.deviceLabel} [${d.integrationName}]`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Tailscale Key Expiry",
|
||
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
// Diagnostic-log "source" values (see integrations/*/adapter.ts and dns/adapters/*.ts) to display names.
|
||
const SOURCE_LABELS: Record<string, string> = {
|
||
cloudflare: "Cloudflare",
|
||
loopia: "Loopia",
|
||
pihole: "Pi-hole",
|
||
azure: "Azure DNS",
|
||
cpanel: "cPanel",
|
||
technitium: "Technitium",
|
||
tailscale: "Tailscale",
|
||
proxmox: "Proxmox",
|
||
synology: "Synology",
|
||
semaphore: "Semaphore",
|
||
gitea: "Gitea",
|
||
dockhand: "Dockhand",
|
||
uptimekuma: "Uptime Kuma",
|
||
phpipam: "phpIPAM",
|
||
pbs: "Proxmox Backup Server",
|
||
osticket: "osTicket",
|
||
};
|
||
|
||
export function sourceLabel(source: string): string {
|
||
return SOURCE_LABELS[source] ?? source;
|
||
}
|
||
|
||
export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise<void> {
|
||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Integration Down",
|
||
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
|
||
);
|
||
}
|
||
|
||
export async function notifyIntegrationRecovered(source: string): Promise<void> {
|
||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
|
||
}
|