The data was all being collected (agent last-seen, per-disk usage, Proxmox storage, Synology volume/disk health) but nothing acted on it, so a dead server or a full disk was only noticed by opening the right page. A new health pass runs every 15 minutes (matching the agent's default report interval) and raises one notification when a problem starts and one when it clears: a server's agent silent past a threshold (default 60 min), a server disk / Proxmox storage or root filesystem / Synology volume at or above a usage threshold (default 90%), and a Synology volume or disk that isn't "normal", has bad SMART, bad sectors past the threshold, or life remaining below it. Both thresholds and an on/off toggle live under Settings -> Notifications. The parts that make this trustworthy rather than noisy: - A problem is keyed by identity, so it alerts once and not every run; a shared Proxmox storage listed by every node is one problem, not one per node. - Active problems persist across restarts, so a rebuild doesn't re-alert everything already known. - If a source can't be read on a given run (Proxmox/Synology unreachable, one node lacking privileges) its existing problems are held, not reported "cleared" and then re-alerted when it comes back — the integration-failure alert already owns "the integration is down". - For 20 minutes after startup server-derived problems are held too: agents couldn't report while the app was down, so judging them then would report every server offline after any restart. - An offline server's disk figures are stale and are not judged; a server that never reported has no agent and raises nothing. - Tracking continues while the toggle is off (only sending is gated), so turning it back on doesn't dump every long-standing problem. Timestamps without a zone (SQLite's format) are read as UTC; the test runs on a UTC+2 machine, where reading them as local time gives a different answer. Verified with 32 checks: the evaluation rules and the state diff as pure functions (exact thresholds, the proxmox:1 vs proxmox:10 prefix trap, the flapping sequence), then a whole pass against a real Proxmox adapter talking to a fake HTTPS cluster (one node returning 403, the whole API down, a shared storage on two nodes, a node that recovers), a webhook receiver, the real DB, and the persisted state. Not exercised end-to-end: the Synology collection path — its rules are tested on data shaped exactly like the adapter's output types, but I did not stand up a fake DSM. Real dev database mtime untouched. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
344 lines
15 KiB
TypeScript
344 lines
15 KiB
TypeScript
import nodemailer from "nodemailer";
|
||
import { getSettings, type NotificationEvents } from "./settingsStore.js";
|
||
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
|
||
|
||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||
|
||
async function sendGotify(title: string, message: string) {
|
||
const { gotify } = await getSettings();
|
||
if (!gotify.enabled || !gotify.url || !gotify.token) return;
|
||
const base = gotify.url.replace(/\/$/, "");
|
||
try {
|
||
const res = await fetch(`${base}/message?token=${encodeURIComponent(gotify.token)}`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({ title, message, priority: gotify.priority ?? 5 }),
|
||
});
|
||
if (!res.ok) console.error(`[notify] Gotify error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] Gotify failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendNtfy(title: string, message: string) {
|
||
const { ntfy } = await getSettings();
|
||
if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return;
|
||
const base = ntfy.url.replace(/\/$/, "");
|
||
const headers: Record<string, string> = {
|
||
"Content-Type": "text/plain",
|
||
Title: title,
|
||
Priority: String(ntfy.priority ?? 3),
|
||
};
|
||
if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`;
|
||
try {
|
||
const res = await fetch(`${base}/${encodeURIComponent(ntfy.topic)}`, { method: "POST", headers, body: message });
|
||
if (!res.ok) console.error(`[notify] ntfy error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] ntfy failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendSmtp(title: string, message: string) {
|
||
const { smtp } = await getSettings();
|
||
if (!smtp.enabled || !smtp.host || !smtp.to || !smtp.from) return;
|
||
try {
|
||
const transporter = nodemailer.createTransport({
|
||
host: smtp.host,
|
||
port: Number(smtp.port) || 587,
|
||
secure: !!smtp.secure,
|
||
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
|
||
});
|
||
await transporter.sendMail({ from: smtp.from, to: smtp.to, subject: title, text: message });
|
||
} catch (err) {
|
||
console.error("[notify] SMTP failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendWebhook(title: string, message: string) {
|
||
const { webhook } = await getSettings();
|
||
if (!webhook.enabled || !webhook.url) return;
|
||
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
||
if (webhook.secret) headers["X-Webhook-Secret"] = webhook.secret;
|
||
try {
|
||
const res = await fetch(webhook.url, {
|
||
method: "POST",
|
||
headers,
|
||
body: JSON.stringify({ content: `**${title}**\n${message}` }),
|
||
});
|
||
if (!res.ok) console.error(`[notify] Webhook error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] Webhook failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendToChannels(title: string, message: string): Promise<void> {
|
||
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message)]);
|
||
}
|
||
|
||
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
|
||
const parts = new Intl.DateTimeFormat("en-GB", { timeZone: timezone, hour: "2-digit", minute: "2-digit", hour12: false }).formatToParts(
|
||
new Date(),
|
||
);
|
||
return {
|
||
hh: Number(parts.find((p) => p.type === "hour")?.value ?? "0"),
|
||
mm: Number(parts.find((p) => p.type === "minute")?.value ?? "0"),
|
||
};
|
||
}
|
||
|
||
function toMinutes(hhmm: string): number {
|
||
const [h, m] = hhmm.split(":").map(Number);
|
||
return (Number.isFinite(h) ? h : 0) * 60 + (Number.isFinite(m) ? m : 0);
|
||
}
|
||
|
||
async function isQuietHoursActive(): Promise<boolean> {
|
||
const { quietHours, notifications } = await getSettings();
|
||
if (!quietHours.enabled) return false;
|
||
const { hh, mm } = currentTimeInZone(notifications.timezone);
|
||
const nowMin = hh * 60 + mm;
|
||
const startMin = toMinutes(quietHours.start);
|
||
const endMin = toMinutes(quietHours.end);
|
||
if (startMin === endMin) return false; // zero-length window — treat as always off
|
||
// A same-day window (e.g. 09:00 -> 17:00) is a simple range; a window that
|
||
// wraps past midnight (e.g. 22:00 -> 07:00, the common case) means "active"
|
||
// is everything outside [end, start) instead.
|
||
return startMin < endMin ? nowMin >= startMin && nowMin < endMin : nowMin >= startMin || nowMin < endMin;
|
||
}
|
||
|
||
export async function notify(title: string, message: string): Promise<void> {
|
||
if (await isQuietHoursActive()) {
|
||
await enqueueNotification(title, message);
|
||
return;
|
||
}
|
||
await sendToChannels(title, message);
|
||
}
|
||
|
||
/** Called by the daily scheduler at quiet-hours end (or manually) — sends everything queued during the window as one digest, then clears it. */
|
||
export async function flushQuietHoursQueue(): Promise<number> {
|
||
const queued = await listQueuedNotifications();
|
||
if (queued.length === 0) return 0;
|
||
const sections = queued.map((q) => `— ${q.title} —\n${q.message}`);
|
||
await sendToChannels(
|
||
"Homelab Manager — Notifications from quiet hours",
|
||
`${queued.length} notification${queued.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
|
||
);
|
||
await clearQueuedNotifications();
|
||
return queued.length;
|
||
}
|
||
|
||
// ─── Test senders — throw on failure so the route can report it ────────────
|
||
|
||
export async function testGotify(cfg: { url: string; token: string; priority?: number }): Promise<void> {
|
||
const base = cfg.url.replace(/\/$/, "");
|
||
const res = await fetch(`${base}/message?token=${encodeURIComponent(cfg.token)}`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
title: "Homelab Manager — Test",
|
||
message: "Gotify notifications are working correctly.",
|
||
priority: cfg.priority ?? 5,
|
||
}),
|
||
});
|
||
if (!res.ok) throw new Error(`Gotify returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise<void> {
|
||
const base = cfg.url.replace(/\/$/, "");
|
||
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: "Homelab Manager — Test", Priority: String(cfg.priority ?? 3) };
|
||
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
|
||
const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, {
|
||
method: "POST",
|
||
headers,
|
||
body: "ntfy notifications are working correctly.",
|
||
});
|
||
if (!res.ok) throw new Error(`ntfy returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
export async function testSmtp(cfg: {
|
||
host: string;
|
||
port?: number;
|
||
secure?: boolean;
|
||
username?: string;
|
||
password?: string;
|
||
from: string;
|
||
to: string;
|
||
}): Promise<void> {
|
||
const transporter = nodemailer.createTransport({
|
||
host: cfg.host,
|
||
port: Number(cfg.port) || 587,
|
||
secure: !!cfg.secure,
|
||
auth: cfg.username ? { user: cfg.username, pass: cfg.password } : undefined,
|
||
});
|
||
await transporter.verify();
|
||
await transporter.sendMail({ from: cfg.from, to: cfg.to, subject: "Homelab Manager — Test", text: "SMTP notifications are working correctly." });
|
||
}
|
||
|
||
export async function testWebhook(cfg: { url: string; secret?: string }): Promise<void> {
|
||
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
||
if (cfg.secret) headers["X-Webhook-Secret"] = cfg.secret;
|
||
const res = await fetch(cfg.url, {
|
||
method: "POST",
|
||
headers,
|
||
body: JSON.stringify({ content: "**Homelab Manager — Test**\nWebhook notifications are working correctly." }),
|
||
});
|
||
if (!res.ok) throw new Error(`Webhook returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
// ─── Event helpers ───────────────────────────────────────────────────────────
|
||
|
||
async function eventEnabled(key: keyof NotificationEvents): Promise<boolean> {
|
||
const { notifications } = await getSettings();
|
||
return notifications[key] !== false;
|
||
}
|
||
|
||
export async function notifyDnsRecordAdded(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsAdd"))) return;
|
||
await notify("DNS Record Added", `[${providerLabel}] ${zoneName}\n+ ${record.type} ${record.name} → ${record.content}`);
|
||
}
|
||
|
||
export async function notifyDnsRecordUpdated(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsUpdate"))) return;
|
||
await notify("DNS Record Updated", `[${providerLabel}] ${zoneName}\n✎ ${record.type} ${record.name} → ${record.content}`);
|
||
}
|
||
|
||
export async function notifyDnsRecordDeleted(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsDelete"))) return;
|
||
await notify("DNS Record Deleted", `[${providerLabel}] ${zoneName}\n− ${record.type} ${record.name} ${record.content}`);
|
||
}
|
||
|
||
export async function notifySecretExpiry(
|
||
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number }[],
|
||
checkFailures: { name: string; host: string; port: number; error?: string }[] = [],
|
||
): Promise<void> {
|
||
if (expiring.length === 0 && checkFailures.length === 0) return;
|
||
if (!(await eventEnabled("secretCheck"))) return;
|
||
const sections: string[] = [];
|
||
if (expiring.length > 0) {
|
||
const lines = expiring.map((s) => (s.status === "expired" ? `✕ EXPIRED — ${s.name}` : `⚠ ${s.daysLeft}d left — ${s.name}`));
|
||
sections.push(`${expiring.length} secret${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`);
|
||
}
|
||
if (checkFailures.length > 0) {
|
||
const lines = checkFailures.map((f) => `⚠ ${f.name} (${f.host}:${f.port}) — ${f.error ?? "check failed"}`);
|
||
sections.push(
|
||
`Couldn't read the live certificate for ${checkFailures.length} monitored secret${checkFailures.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"));
|
||
}
|
||
|
||
export async function notifyDockerUpdates(
|
||
updatesAvailable: { integrationName: string; containerName: string; environmentName: string; newerVersion: string | null }[],
|
||
): Promise<void> {
|
||
if (updatesAvailable.length === 0) return;
|
||
if (!(await eventEnabled("dockerUpdateCheck"))) return;
|
||
const lines = updatesAvailable.map(
|
||
(u) => `${u.containerName} [${u.environmentName}, ${u.integrationName}]${u.newerVersion ? ` → ${u.newerVersion}` : ""}`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Docker Updates Available",
|
||
`${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyProxmoxBackupFailure(
|
||
failures: { integrationName: string; node: string; guestId: string | null; status: string }[],
|
||
): Promise<void> {
|
||
if (failures.length === 0) return;
|
||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||
const lines = failures.map(
|
||
(f) => `${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} [${f.integrationName}]: ${f.status}`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Proxmox Backup Failed",
|
||
`${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyProxmoxUncoveredGuests(
|
||
uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[],
|
||
): Promise<void> {
|
||
if (uncovered.length === 0) return;
|
||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||
const lines = uncovered.map((u) => `${u.guestName} (#${u.vmid}) on ${u.node} [${u.integrationName}]`);
|
||
await notify(
|
||
"Homelab Manager — Proxmox Guests Without Backup Coverage",
|
||
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyHealthIssues(issues: { message: string }[]): Promise<void> {
|
||
if (issues.length === 0) return;
|
||
if (!(await eventEnabled("healthAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Health Alert",
|
||
`${issues.length} new issue${issues.length !== 1 ? "s" : ""}:\n\n${issues.map((i) => `⚠ ${i.message}`).join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyHealthRecovered(resolved: { message: string }[]): Promise<void> {
|
||
if (resolved.length === 0) return;
|
||
if (!(await eventEnabled("healthAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Health Recovered",
|
||
`${resolved.length} issue${resolved.length !== 1 ? "s" : ""} cleared:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyTailscaleKeyExpiry(
|
||
expiring: { integrationName: string; deviceLabel: string; daysLeft: number }[],
|
||
): Promise<void> {
|
||
if (expiring.length === 0) return;
|
||
if (!(await eventEnabled("tailscaleKeyCheck"))) return;
|
||
const lines = expiring.map(
|
||
(d) => `${d.daysLeft < 0 ? "✕ EXPIRED" : `⚠ ${d.daysLeft}d left`} — ${d.deviceLabel} [${d.integrationName}]`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Tailscale Key Expiry",
|
||
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
// Diagnostic-log "source" values (see integrations/*/adapter.ts and dns/adapters/*.ts) to display names.
|
||
const SOURCE_LABELS: Record<string, string> = {
|
||
cloudflare: "Cloudflare",
|
||
loopia: "Loopia",
|
||
pihole: "Pi-hole",
|
||
azure: "Azure DNS",
|
||
cpanel: "cPanel",
|
||
technitium: "Technitium",
|
||
tailscale: "Tailscale",
|
||
proxmox: "Proxmox",
|
||
synology: "Synology",
|
||
semaphore: "Semaphore",
|
||
gitea: "Gitea",
|
||
dockhand: "Dockhand",
|
||
};
|
||
|
||
function sourceLabel(source: string): string {
|
||
return SOURCE_LABELS[source] ?? source;
|
||
}
|
||
|
||
export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise<void> {
|
||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Integration Down",
|
||
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
|
||
);
|
||
}
|
||
|
||
export async function notifyIntegrationRecovered(source: string): Promise<void> {
|
||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
|
||
}
|