- Dependencies: servers and integrations can depend on each other; alerts are folded under the root cause, and a failure explained by something down isn't announced - Web push: hand-written VAPID and RFC 8291 encryption, per-device subscriptions, a fifth notification channel, service worker handlers - Inventory: maker/model/serial from the agents fill blank fields, "Refresh from agent", QR label printing and ?asset= deep links - Windows agent (v4) reports failed services, disk health, Storage Spaces pools and hardware identity; Linux agent (v5) reports hardware identity Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
461 lines
21 KiB
TypeScript
461 lines
21 KiB
TypeScript
import nodemailer from "nodemailer";
|
||
import { getSettings, type NotificationEvents } from "./settingsStore.js";
|
||
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
|
||
import { emitEvent } from "./eventWebhooks.js";
|
||
import { sendPush, subscriptionCount } from "./webPush.js";
|
||
|
||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||
|
||
async function sendGotify(title: string, message: string) {
|
||
const { gotify } = await getSettings();
|
||
if (!gotify.enabled || !gotify.url || !gotify.token) return;
|
||
const base = gotify.url.replace(/\/$/, "");
|
||
try {
|
||
const res = await fetch(`${base}/message?token=${encodeURIComponent(gotify.token)}`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({ title, message, priority: gotify.priority ?? 5 }),
|
||
});
|
||
if (!res.ok) console.error(`[notify] Gotify error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] Gotify failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendNtfy(title: string, message: string) {
|
||
const { ntfy } = await getSettings();
|
||
if (!ntfy.enabled || !ntfy.url || !ntfy.topic) return;
|
||
const base = ntfy.url.replace(/\/$/, "");
|
||
const headers: Record<string, string> = {
|
||
"Content-Type": "text/plain",
|
||
Title: title,
|
||
Priority: String(ntfy.priority ?? 3),
|
||
};
|
||
if (ntfy.token) headers.Authorization = `Bearer ${ntfy.token}`;
|
||
try {
|
||
const res = await fetch(`${base}/${encodeURIComponent(ntfy.topic)}`, { method: "POST", headers, body: message });
|
||
if (!res.ok) console.error(`[notify] ntfy error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] ntfy failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendSmtp(title: string, message: string) {
|
||
const { smtp } = await getSettings();
|
||
if (!smtp.enabled || !smtp.host || !smtp.to || !smtp.from) return;
|
||
try {
|
||
const transporter = nodemailer.createTransport({
|
||
host: smtp.host,
|
||
port: Number(smtp.port) || 587,
|
||
secure: !!smtp.secure,
|
||
auth: smtp.username ? { user: smtp.username, pass: smtp.password } : undefined,
|
||
});
|
||
await transporter.sendMail({ from: smtp.from, to: smtp.to, subject: title, text: message });
|
||
} catch (err) {
|
||
console.error("[notify] SMTP failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendWebhook(title: string, message: string) {
|
||
const { webhook } = await getSettings();
|
||
if (!webhook.enabled || !webhook.url) return;
|
||
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
||
if (webhook.secret) headers["X-Webhook-Secret"] = webhook.secret;
|
||
try {
|
||
const res = await fetch(webhook.url, {
|
||
method: "POST",
|
||
headers,
|
||
body: JSON.stringify({ content: `**${title}**\n${message}` }),
|
||
});
|
||
if (!res.ok) console.error(`[notify] Webhook error ${res.status}: ${await res.text().catch(() => "")}`);
|
||
} catch (err) {
|
||
console.error("[notify] Webhook failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendWebPush(title: string, message: string) {
|
||
const { webPush } = await getSettings();
|
||
if (!webPush.enabled) return;
|
||
try {
|
||
await sendPush({ title, body: message });
|
||
} catch (err) {
|
||
console.error("[notify] Web push failed:", err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
async function sendToChannels(title: string, message: string): Promise<void> {
|
||
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message), sendWebPush(title, message)]);
|
||
}
|
||
|
||
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
|
||
const parts = new Intl.DateTimeFormat("en-GB", { timeZone: timezone, hour: "2-digit", minute: "2-digit", hour12: false }).formatToParts(
|
||
new Date(),
|
||
);
|
||
return {
|
||
hh: Number(parts.find((p) => p.type === "hour")?.value ?? "0"),
|
||
mm: Number(parts.find((p) => p.type === "minute")?.value ?? "0"),
|
||
};
|
||
}
|
||
|
||
function toMinutes(hhmm: string): number {
|
||
const [h, m] = hhmm.split(":").map(Number);
|
||
return (Number.isFinite(h) ? h : 0) * 60 + (Number.isFinite(m) ? m : 0);
|
||
}
|
||
|
||
async function isQuietHoursActive(): Promise<boolean> {
|
||
const { quietHours, notifications } = await getSettings();
|
||
if (!quietHours.enabled) return false;
|
||
const { hh, mm } = currentTimeInZone(notifications.timezone);
|
||
const nowMin = hh * 60 + mm;
|
||
const startMin = toMinutes(quietHours.start);
|
||
const endMin = toMinutes(quietHours.end);
|
||
if (startMin === endMin) return false; // zero-length window — treat as always off
|
||
// A same-day window (e.g. 09:00 -> 17:00) is a simple range; a window that
|
||
// wraps past midnight (e.g. 22:00 -> 07:00, the common case) means "active"
|
||
// is everything outside [end, start) instead.
|
||
return startMin < endMin ? nowMin >= startMin && nowMin < endMin : nowMin >= startMin || nowMin < endMin;
|
||
}
|
||
|
||
/** Sends at once through every channel that is switched on, whatever the time — for something a person just asked for. */
|
||
export async function notifyNow(title: string, message: string): Promise<void> {
|
||
await sendToChannels(title, message);
|
||
}
|
||
|
||
/** How many channels are switched on and filled in enough to send, so a "send it now" can say when there is nowhere to send to. */
|
||
export async function enabledChannelCount(): Promise<number> {
|
||
const { gotify, ntfy, smtp, webhook, webPush } = await getSettings();
|
||
return [
|
||
webPush.enabled && (await subscriptionCount()) > 0,
|
||
gotify.enabled && gotify.url && gotify.token,
|
||
ntfy.enabled && ntfy.url && ntfy.topic,
|
||
smtp.enabled && smtp.host && smtp.to && smtp.from,
|
||
webhook.enabled && webhook.url,
|
||
].filter(Boolean).length;
|
||
}
|
||
|
||
export async function notify(title: string, message: string): Promise<void> {
|
||
if (await isQuietHoursActive()) {
|
||
await enqueueNotification(title, message);
|
||
return;
|
||
}
|
||
await sendToChannels(title, message);
|
||
}
|
||
|
||
/** Called by the daily scheduler at quiet-hours end (or manually) — sends everything queued during the window as one digest, then clears it. */
|
||
export async function flushQuietHoursQueue(): Promise<number> {
|
||
const queued = await listQueuedNotifications();
|
||
if (queued.length === 0) return 0;
|
||
const sections = queued.map((q) => `— ${q.title} —\n${q.message}`);
|
||
await sendToChannels(
|
||
"Homelab Manager — Notifications from quiet hours",
|
||
`${queued.length} notification${queued.length !== 1 ? "s were" : " was"} held during quiet hours:\n\n${sections.join("\n\n")}`,
|
||
);
|
||
await clearQueuedNotifications();
|
||
return queued.length;
|
||
}
|
||
|
||
// ─── Test senders — throw on failure so the route can report it ────────────
|
||
|
||
export async function testGotify(cfg: { url: string; token: string; priority?: number }): Promise<void> {
|
||
const base = cfg.url.replace(/\/$/, "");
|
||
const res = await fetch(`${base}/message?token=${encodeURIComponent(cfg.token)}`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
title: "Homelab Manager — Test",
|
||
message: "Gotify notifications are working correctly.",
|
||
priority: cfg.priority ?? 5,
|
||
}),
|
||
});
|
||
if (!res.ok) throw new Error(`Gotify returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
export async function testNtfy(cfg: { url: string; topic: string; token?: string; priority?: number }): Promise<void> {
|
||
const base = cfg.url.replace(/\/$/, "");
|
||
const headers: Record<string, string> = { "Content-Type": "text/plain", Title: "Homelab Manager — Test", Priority: String(cfg.priority ?? 3) };
|
||
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
|
||
const res = await fetch(`${base}/${encodeURIComponent(cfg.topic)}`, {
|
||
method: "POST",
|
||
headers,
|
||
body: "ntfy notifications are working correctly.",
|
||
});
|
||
if (!res.ok) throw new Error(`ntfy returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
export async function testSmtp(cfg: {
|
||
host: string;
|
||
port?: number;
|
||
secure?: boolean;
|
||
username?: string;
|
||
password?: string;
|
||
from: string;
|
||
to: string;
|
||
}): Promise<void> {
|
||
const transporter = nodemailer.createTransport({
|
||
host: cfg.host,
|
||
port: Number(cfg.port) || 587,
|
||
secure: !!cfg.secure,
|
||
auth: cfg.username ? { user: cfg.username, pass: cfg.password } : undefined,
|
||
});
|
||
await transporter.verify();
|
||
await transporter.sendMail({ from: cfg.from, to: cfg.to, subject: "Homelab Manager — Test", text: "SMTP notifications are working correctly." });
|
||
}
|
||
|
||
export async function testWebhook(cfg: { url: string; secret?: string }): Promise<void> {
|
||
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
||
if (cfg.secret) headers["X-Webhook-Secret"] = cfg.secret;
|
||
const res = await fetch(cfg.url, {
|
||
method: "POST",
|
||
headers,
|
||
body: JSON.stringify({ content: "**Homelab Manager — Test**\nWebhook notifications are working correctly." }),
|
||
});
|
||
if (!res.ok) throw new Error(`Webhook returned ${res.status}: ${await res.text().catch(() => "")}`);
|
||
}
|
||
|
||
// ─── Event helpers ───────────────────────────────────────────────────────────
|
||
|
||
async function eventEnabled(key: keyof NotificationEvents): Promise<boolean> {
|
||
const { notifications } = await getSettings();
|
||
return notifications[key] !== false;
|
||
}
|
||
|
||
export async function notifyDnsRecordAdded(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsAdd"))) return;
|
||
await notify("DNS Record Added", `[${providerLabel}] ${zoneName}\n+ ${record.type} ${record.name} → ${record.content}`);
|
||
}
|
||
|
||
export async function notifyDnsRecordUpdated(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsUpdate"))) return;
|
||
await notify("DNS Record Updated", `[${providerLabel}] ${zoneName}\n✎ ${record.type} ${record.name} → ${record.content}`);
|
||
}
|
||
|
||
export async function notifyDnsRecordDeleted(
|
||
providerLabel: string,
|
||
zoneName: string,
|
||
record: { type: string; name: string; content: string },
|
||
): Promise<void> {
|
||
if (!(await eventEnabled("dnsDelete"))) return;
|
||
await notify("DNS Record Deleted", `[${providerLabel}] ${zoneName}\n− ${record.type} ${record.name} ${record.content}`);
|
||
}
|
||
|
||
export async function notifySecretExpiry(
|
||
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number }[],
|
||
checkFailures: { name: string; host: string; port: number; error?: string }[] = [],
|
||
): Promise<void> {
|
||
if (expiring.length === 0 && checkFailures.length === 0) return;
|
||
if (!(await eventEnabled("secretCheck"))) return;
|
||
const sections: string[] = [];
|
||
if (expiring.length > 0) {
|
||
const lines = expiring.map((s) => (s.status === "expired" ? `✕ EXPIRED — ${s.name}` : `⚠ ${s.daysLeft}d left — ${s.name}`));
|
||
sections.push(`${expiring.length} secret${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`);
|
||
}
|
||
if (checkFailures.length > 0) {
|
||
const lines = checkFailures.map((f) => `⚠ ${f.name} (${f.host}:${f.port}) — ${f.error ?? "check failed"}`);
|
||
sections.push(
|
||
`Couldn't read the live certificate for ${checkFailures.length} monitored secret${checkFailures.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
await notify("Homelab Manager — Secrets Alert", sections.join("\n\n"));
|
||
}
|
||
|
||
export async function notifyDockerUpdates(
|
||
updatesAvailable: { integrationName: string; containerName: string; environmentName: string; newerVersion: string | null }[],
|
||
): Promise<void> {
|
||
if (updatesAvailable.length === 0) return;
|
||
if (!(await eventEnabled("dockerUpdateCheck"))) return;
|
||
const lines = updatesAvailable.map(
|
||
(u) => `${u.containerName} [${u.environmentName}, ${u.integrationName}]${u.newerVersion ? ` → ${u.newerVersion}` : ""}`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Docker Updates Available",
|
||
`${updatesAvailable.length} container${updatesAvailable.length !== 1 ? "s have" : " has"} an image update available:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyProxmoxBackupFailure(
|
||
failures: { integrationName: string; node: string; guestId: string | null; status: string }[],
|
||
): Promise<void> {
|
||
if (failures.length === 0) return;
|
||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||
const lines = failures.map(
|
||
(f) => `${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} [${f.integrationName}]: ${f.status}`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Proxmox Backup Failed",
|
||
`${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyProxmoxUncoveredGuests(
|
||
uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[],
|
||
): Promise<void> {
|
||
if (uncovered.length === 0) return;
|
||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||
const lines = uncovered.map((u) => `${u.guestName} (#${u.vmid}) on ${u.node} [${u.integrationName}]`);
|
||
await notify(
|
||
"Homelab Manager — Proxmox Guests Without Backup Coverage",
|
||
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyStaleBackups(stale: { message: string }[]): Promise<void> {
|
||
if (stale.length === 0) return;
|
||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||
await notify(
|
||
"Homelab Manager — Backups Gone Stale",
|
||
`${stale.length} guest${stale.length !== 1 ? "s have" : " has"} no recent backup:\n\n${stale.map((s) => s.message).join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyUpdateRunFinished(run: { serverName: string; templateName: string; ok: boolean; message: string }): Promise<void> {
|
||
if (!(await eventEnabled("updateRunResults"))) return;
|
||
await notify(
|
||
run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`,
|
||
`"${run.templateName}" on ${run.serverName}: ${run.message}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyPbsVerificationFailed(
|
||
failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[],
|
||
): Promise<void> {
|
||
if (failures.length === 0) return;
|
||
if (!(await eventEnabled("pbsVerificationCheck"))) return;
|
||
const lines = failures.map((f) =>
|
||
f.error
|
||
? `${f.datastore} [${f.integrationName}]: couldn't be read — ${f.error}`
|
||
: `${f.datastore} [${f.integrationName}]: ${f.failedCount} snapshot${f.failedCount !== 1 ? "s" : ""} failed verification`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Proxmox Backup Server Verification Failed",
|
||
`${failures.length} datastore${failures.length !== 1 ? "s have" : " has"} a problem:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyHealthIssues(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
|
||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||
if (issues.length === 0) return;
|
||
if (!(await eventEnabled("healthAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Health Alert",
|
||
`${issues.length} new issue${issues.length !== 1 ? "s" : ""}:\n\n${issues.map((i) => `⚠ ${i.message}`).join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyHealthRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
|
||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||
if (resolved.length === 0) return;
|
||
if (!(await eventEnabled("healthAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Health Recovered",
|
||
`${resolved.length} issue${resolved.length !== 1 ? "s" : ""} cleared:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
|
||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||
if (issues.length === 0) return;
|
||
if (!(await eventEnabled("automationAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Automation Failed",
|
||
`${issues.length} run${issues.length !== 1 ? "s" : ""} failed:\n\n${issues.map((i) => `✕ ${i.message}`).join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyAutomationRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
|
||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||
if (resolved.length === 0) return;
|
||
if (!(await eventEnabled("automationAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Automation Recovered",
|
||
`${resolved.length} no longer failing:\n\n${resolved.map((i) => `✓ ${i.message}`).join("\n")}`,
|
||
);
|
||
}
|
||
|
||
export async function notifyDomainExpiry(
|
||
expiring: { name: string; status: "expired" | "expiring"; daysLeft: number; expiresAt: string; registrar: string | null }[],
|
||
staleChecks: { name: string; error: string; lastOkAt: string | null }[] = [],
|
||
): Promise<void> {
|
||
if (expiring.length === 0 && staleChecks.length === 0) return;
|
||
if (!(await eventEnabled("domainExpiryCheck"))) return;
|
||
const sections: string[] = [];
|
||
if (expiring.length > 0) {
|
||
const lines = expiring.map((d) => {
|
||
const registrar = d.registrar ? ` [${d.registrar}]` : "";
|
||
return d.status === "expired"
|
||
? `✕ EXPIRED ${d.expiresAt} — ${d.name}${registrar}`
|
||
: `⚠ ${d.daysLeft}d left (${d.expiresAt}) — ${d.name}${registrar}`;
|
||
});
|
||
sections.push(`${expiring.length} domain${expiring.length !== 1 ? "s" : ""} need renewing:\n\n${lines.join("\n")}`);
|
||
}
|
||
if (staleChecks.length > 0) {
|
||
const lines = staleChecks.map((s) => `⚠ ${s.name} — ${s.error}${s.lastOkAt ? ` (last read ${s.lastOkAt.slice(0, 10)})` : " (never read)"}`);
|
||
sections.push(
|
||
`Couldn't refresh the registration for ${staleChecks.length} domain${staleChecks.length !== 1 ? "s" : ""} — the expiry shown may be stale:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
await notify("Homelab Manager — Domain Expiry", sections.join("\n\n"));
|
||
}
|
||
|
||
export async function notifyTailscaleKeyExpiry(
|
||
expiring: { integrationName: string; deviceLabel: string; daysLeft: number }[],
|
||
): Promise<void> {
|
||
if (expiring.length === 0) return;
|
||
if (!(await eventEnabled("tailscaleKeyCheck"))) return;
|
||
const lines = expiring.map(
|
||
(d) => `${d.daysLeft < 0 ? "✕ EXPIRED" : `⚠ ${d.daysLeft}d left`} — ${d.deviceLabel} [${d.integrationName}]`,
|
||
);
|
||
await notify(
|
||
"Homelab Manager — Tailscale Key Expiry",
|
||
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
|
||
);
|
||
}
|
||
|
||
// Diagnostic-log "source" values (see integrations/*/adapter.ts and dns/adapters/*.ts) to display names.
|
||
const SOURCE_LABELS: Record<string, string> = {
|
||
cloudflare: "Cloudflare",
|
||
loopia: "Loopia",
|
||
pihole: "Pi-hole",
|
||
azure: "Azure DNS",
|
||
cpanel: "cPanel",
|
||
technitium: "Technitium",
|
||
tailscale: "Tailscale",
|
||
proxmox: "Proxmox",
|
||
synology: "Synology",
|
||
semaphore: "Semaphore",
|
||
gitea: "Gitea",
|
||
dockhand: "Dockhand",
|
||
uptimekuma: "Uptime Kuma",
|
||
phpipam: "phpIPAM",
|
||
pbs: "Proxmox Backup Server",
|
||
osticket: "osTicket",
|
||
};
|
||
|
||
export function sourceLabel(source: string): string {
|
||
return SOURCE_LABELS[source] ?? source;
|
||
}
|
||
|
||
export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise<void> {
|
||
emitEvent("alert.opened", { alert: { key: `integration:${source}`, source, severity: "warning", message: `${sourceLabel(source)} has failed its last ${consecutiveFailures} calls in a row.` } });
|
||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||
await notify(
|
||
"Homelab Manager — Integration Down",
|
||
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
|
||
);
|
||
}
|
||
|
||
export async function notifyIntegrationRecovered(source: string): Promise<void> {
|
||
emitEvent("alert.resolved", { alert: { key: `integration:${source}`, message: `${sourceLabel(source)} is answering again.` } });
|
||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
|
||
}
|