Files
Homelab-manager/server/src/services/notify.ts
T
bobbanandClaude Sonnet 5 1688de3ea2 Alert when a server goes silent, a disk fills up, or a Synology volume degrades
The data was all being collected (agent last-seen, per-disk usage,
Proxmox storage, Synology volume/disk health) but nothing acted on
it, so a dead server or a full disk was only noticed by opening the
right page.

A new health pass runs every 15 minutes (matching the agent's default
report interval) and raises one notification when a problem starts and
one when it clears: a server's agent silent past a threshold (default
60 min), a server disk / Proxmox storage or root filesystem / Synology
volume at or above a usage threshold (default 90%), and a Synology
volume or disk that isn't "normal", has bad SMART, bad sectors past the
threshold, or life remaining below it. Both thresholds and an on/off
toggle live under Settings -> Notifications.

The parts that make this trustworthy rather than noisy:
- A problem is keyed by identity, so it alerts once and not every run;
  a shared Proxmox storage listed by every node is one problem, not
  one per node.
- Active problems persist across restarts, so a rebuild doesn't
  re-alert everything already known.
- If a source can't be read on a given run (Proxmox/Synology
  unreachable, one node lacking privileges) its existing problems are
  held, not reported "cleared" and then re-alerted when it comes back —
  the integration-failure alert already owns "the integration is down".
- For 20 minutes after startup server-derived problems are held too:
  agents couldn't report while the app was down, so judging them then
  would report every server offline after any restart.
- An offline server's disk figures are stale and are not judged; a
  server that never reported has no agent and raises nothing.
- Tracking continues while the toggle is off (only sending is gated),
  so turning it back on doesn't dump every long-standing problem.

Timestamps without a zone (SQLite's format) are read as UTC; the test
runs on a UTC+2 machine, where reading them as local time gives a
different answer.

Verified with 32 checks: the evaluation rules and the state diff as
pure functions (exact thresholds, the proxmox:1 vs proxmox:10 prefix
trap, the flapping sequence), then a whole pass against a real
Proxmox adapter talking to a fake HTTPS cluster (one node returning
403, the whole API down, a shared storage on two nodes, a node that
recovers), a webhook receiver, the real DB, and the persisted state.

Not exercised end-to-end: the Synology collection path — its rules are
tested on data shaped exactly like the adapter's output types, but I
did not stand up a fake DSM. Real dev database mtime untouched.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-26 02:15:19 +02:00

344 lines
15 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import nodemailer from "nodemailer";
import { getSettings, type NotificationEvents } from "./settingsStore.js";
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
async function sendGotify(title: string, message: string) {
const { gotify } = await getSettings();
if (!gotify.enabled || !gotify.url || !gotify.token) return;
const base = gotify.url.replace(/\/$/, "");
try {
const res = await fetch(`${base}/message?token=${encodeURIComponent(gotify.token)}`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ title, message, priority: gotify.priority ?? 5 }),
});
if (!res.ok) console.error(`[notify] Gotify error ${res.status}: ${await res.text().catch(() => "")}`);
} catch (err) {
console.error("[notify] Gotify failed:", err instanceof Error ? err.message : err);
}
}
async function sendNtfy(title: string, message: string) {
const { ntfy } = await getSettings();
if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return;
const base = ntfy.url.replace(/\/$/, "");
const headers: Record<string, string> = {
"Content-Type": "text/plain",
Title: title,
Priority: String(ntfy.priority ?? 3),
};
if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`;
try {
const res = await fetch(`${base}/${encodeURIComponent(ntfy.topic)}`, { method: "POST", headers, body: message });
if (!res.ok) console.error(`[notify] ntfy error ${res.status}: ${await res.text().catch(() => "")}`);
} catch (err) {
console.error("[notify] ntfy failed:", err instanceof Error ? err.message : err);
}
}
async function sendSmtp(title: string, message: string) {
const { smtp } = await getSettings();
if (!smtp.enabled || !smtp.host || !smtp.to || !smtp.from) return;
try {
const transporter = nodemailer.createTransport({
host: smtp.host,
port: Number(smtp.port) || 587,
secure: !!smtp.secure,
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
});
await transporter.sendMail({ from: smtp.from, to: smtp.to, subject: title, text: message });
} catch (err) {
console.error("[notify] SMTP failed:", err instanceof Error ? err.message : err);
}
}
async function sendWebhook(title: string, message: string) {
const { webhook } = await getSettings();
if (!webhook.enabled || !webhook.url) return;
const headers: Record<string, string> = { "Content-Type": "application/json" };
if (webhook.secret) headers["X-Webhook-Secret"] = webhook.secret;
try {
const res = await fetch(webhook.url, {
method: "POST",
headers,
body: JSON.stringify({ content: `**${title}**\n${message}` }),
});
if (!res.ok) console.error(`[notify] Webhook error ${res.status}: ${await res.text().catch(() => "")}`);
} catch (err) {
console.error("[notify] Webhook failed:", err instanceof Error ? err.message : err);
}
}
async function sendToChannels(title: string, message: string): Promise<void> {
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message)]);
}
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
const parts = new Intl.DateTimeFormat("en-GB", { timeZone: timezone, hour: "2-digit", minute: "2-digit", hour12: false }).formatToParts(
new Date(),
);
return {
hh: Number(parts.find((p) => p.type === "hour")?.value ?? "0"),
mm: Number(parts.find((p) => p.type === "minute")?.value ?? "0"),
};
}
function toMinutes(hhmm: string): number {
const [h, m] = hhmm.split(":").map(Number);
return (Number.isFinite(h) ? h : 0) * 60 + (Number.isFinite(m) ? m : 0);
}
async function isQuietHoursActive(): Promise<boolean> {
const { quietHours, notifications } = await getSettings();
if (!quietHours.enabled) return false;
const { hh, mm } = currentTimeInZone(notifications.timezone);
const nowMin = hh * 60 + mm;
const startMin = toMinutes(quietHours.start);
const endMin = toMinutes(quietHours.end);
if (startMin === endMin) return false; // zero-length window — treat as always off
// A same-day window (e.g. 09:00 -> 17:00) is a simple range; a window that
// wraps past midnight (e.g. 22:00 -> 07:00, the common case) means "active"
// is everything outside [end, start) instead.
return startMin < endMin ? nowMin >= startMin && nowMin < endMin : nowMin >= startMin || nowMin < endMin;
}
export async function notify(title: string, message: string): Promise<void> {
if (await isQuietHoursActive()) {
await enqueueNotification(title, message);
return;
}
await sendToChannels(title, message);
}
/** Called by the daily scheduler at quiet-hours end (or manually) — sends everything queued during the window as one digest, then clears it. */
export async function flushQuietHoursQueue(): Promise<number> {
const queued = await listQueuedNotifications();
if (queued.length === 0) return 0;
const sections = queued.map((q) => `— ${q.title} —\n${q.message}`);
await sendToChannels(
"Homelab Manager — Notifications from quiet hours",
`${queued.length} notification${queued.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
);
await clearQueuedNotifications();
return queued.length;
}
// ─── Test senders — throw on failure so the route can report it ────────────
export async function testGotify(cfg: { url: string; token: string; priority?: number }): Promise<void> {
const base = cfg.url.replace(/\/$/, "");
const res = await fetch(`${base}/message?token=${encodeURIComponent(cfg.token)}`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
title: "Homelab Manager — Test",
message: "Gotify notifications are working correctly.",
priority: cfg.priority ?? 5,
}),
});
if (!res.ok) throw new Error(`Gotify returned ${res.status}: ${await res.text().catch(() => "")}`);
}
export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise<void> {
const base = cfg.url.replace(/\/$/, "");
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: "Homelab Manager — Test", Priority: String(cfg.priority ?? 3) };
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, {
method: "POST",
headers,
body: "ntfy notifications are working correctly.",
});
if (!res.ok) throw new Error(`ntfy returned ${res.status}: ${await res.text().catch(() => "")}`);
}
export async function testSmtp(cfg: {
host: string;
port?: number;
secure?: boolean;
username?: string;
password?: string;
from: string;
to: string;
}): Promise<void> {
const transporter = nodemailer.createTransport({
host: cfg.host,
port: Number(cfg.port) || 587,
secure: !!cfg.secure,
auth: cfg.username ? { user: cfg.username, pass: cfg.password } : undefined,
});
await transporter.verify();
await transporter.sendMail({ from: cfg.from, to: cfg.to, subject: "Homelab Manager — Test", text: "SMTP notifications are working correctly." });
}
export async function testWebhook(cfg: { url: string; secret?: string }): Promise<void> {
const headers: Record<string, string> = { "Content-Type": "application/json" };
if (cfg.secret) headers["X-Webhook-Secret"] = cfg.secret;
const res = await fetch(cfg.url, {
method: "POST",
headers,
body: JSON.stringify({ content: "**Homelab Manager — Test**\nWebhook notifications are working correctly." }),
});
if (!res.ok) throw new Error(`Webhook returned ${res.status}: ${await res.text().catch(() => "")}`);
}
// ─── Event helpers ───────────────────────────────────────────────────────────
async function eventEnabled(key: keyof NotificationEvents): Promise<boolean> {
const { notifications } = await getSettings();
return notifications[key] !== false;
}
export async function notifyDnsRecordAdded(
providerLabel: string,
zoneName: string,
record: { type: string; name: string; content: string },
): Promise<void> {
if (!(await eventEnabled("dnsAdd"))) return;
await notify("DNS Record Added", `[${providerLabel}] ${zoneName}\n+ ${record.type} ${record.name} → ${record.content}`);
}
export async function notifyDnsRecordUpdated(
providerLabel: string,
zoneName: string,
record: { type: string; name: string; content: string },
): Promise<void> {
if (!(await eventEnabled("dnsUpdate"))) return;
await notify("DNS Record Updated", `[${providerLabel}] ${zoneName}\n✎ ${record.type} ${record.name} → ${record.content}`);
}
export async function notifyDnsRecordDeleted(
providerLabel: string,
zoneName: string,
record: { type: string; name: string; content: string },
): Promise<void> {
if (!(await eventEnabled("dnsDelete"))) return;
await notify("DNS Record Deleted", `[${providerLabel}] ${zoneName}\n− ${record.type} ${record.name} ${record.content}`);
}
export async function notifySecretExpiry(
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number }[],
checkFailures: { name: string; host: string; port: number; error?: string }[] = [],
): Promise<void> {
if (expiring.length === 0 && checkFailures.length === 0) return;
if (!(await eventEnabled("secretCheck"))) return;
const sections: string[] = [];
if (expiring.length > 0) {
const lines = expiring.map((s) => (s.status === "expired" ? `✕ EXPIRED — ${s.name}` : `⚠ ${s.daysLeft}d left — ${s.name}`));
sections.push(`${expiring.length} secret${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`);
}
if (checkFailures.length > 0) {
const lines = checkFailures.map((f) => `⚠ ${f.name} (${f.host}:${f.port}) — ${f.error ?? "check failed"}`);
sections.push(
`Couldn't read the live certificate for ${checkFailures.length} monitored secret${checkFailures.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
);
}
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"));
}
export async function notifyDockerUpdates(
updatesAvailable: { integrationName: string; containerName: string; environmentName: string; newerVersion: string | null }[],
): Promise<void> {
if (updatesAvailable.length === 0) return;
if (!(await eventEnabled("dockerUpdateCheck"))) return;
const lines = updatesAvailable.map(
(u) => `${u.containerName} [${u.environmentName}, ${u.integrationName}]${u.newerVersion ? ` → ${u.newerVersion}` : ""}`,
);
await notify(
"Homelab Manager — Docker Updates Available",
`${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`,
);
}
export async function notifyProxmoxBackupFailure(
failures: { integrationName: string; node: string; guestId: string | null; status: string }[],
): Promise<void> {
if (failures.length === 0) return;
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
const lines = failures.map(
(f) => `${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} [${f.integrationName}]: ${f.status}`,
);
await notify(
"Homelab Manager — Proxmox Backup Failed",
`${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
);
}
export async function notifyProxmoxUncoveredGuests(
uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[],
): Promise<void> {
if (uncovered.length === 0) return;
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
const lines = uncovered.map((u) => `${u.guestName} (#${u.vmid}) on ${u.node} [${u.integrationName}]`);
await notify(
"Homelab Manager — Proxmox Guests Without Backup Coverage",
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
);
}
export async function notifyHealthIssues(issues: { message: string }[]): Promise<void> {
if (issues.length === 0) return;
if (!(await eventEnabled("healthAlerts"))) return;
await notify(
"Homelab Manager — Health Alert",
`${issues.length} new issue${issues.length !== 1 ? "s" : ""}:\n\n${issues.map((i) => `⚠ ${i.message}`).join("\n")}`,
);
}
export async function notifyHealthRecovered(resolved: { message: string }[]): Promise<void> {
if (resolved.length === 0) return;
if (!(await eventEnabled("healthAlerts"))) return;
await notify(
"Homelab Manager — Health Recovered",
`${resolved.length} issue${resolved.length !== 1 ? "s" : ""} cleared:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
);
}
export async function notifyTailscaleKeyExpiry(
expiring: { integrationName: string; deviceLabel: string; daysLeft: number }[],
): Promise<void> {
if (expiring.length === 0) return;
if (!(await eventEnabled("tailscaleKeyCheck"))) return;
const lines = expiring.map(
(d) => `${d.daysLeft < 0 ? "✕ EXPIRED" : `⚠ ${d.daysLeft}d left`} — ${d.deviceLabel} [${d.integrationName}]`,
);
await notify(
"Homelab Manager — Tailscale Key Expiry",
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
);
}
// Diagnostic-log "source" values (see integrations/*/adapter.ts and dns/adapters/*.ts) to display names.
const SOURCE_LABELS: Record<string, string> = {
cloudflare: "Cloudflare",
loopia: "Loopia",
pihole: "Pi-hole",
azure: "Azure DNS",
cpanel: "cPanel",
technitium: "Technitium",
tailscale: "Tailscale",
proxmox: "Proxmox",
synology: "Synology",
semaphore: "Semaphore",
gitea: "Gitea",
dockhand: "Dockhand",
};
function sourceLabel(source: string): string {
return SOURCE_LABELS[source] ?? source;
}
export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise<void> {
if (!(await eventEnabled("integrationFailureAlerts"))) return;
await notify(
"Homelab Manager — Integration Down",
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
);
}
export async function notifyIntegrationRecovered(source: string): Promise<void> {
if (!(await eventEnabled("integrationFailureAlerts"))) return;
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
}