diff --git a/server/src/routes/settings.ts b/server/src/routes/settings.ts index d7a21d6..0596999 100644 --- a/server/src/routes/settings.ts +++ b/server/src/routes/settings.ts @@ -63,6 +63,8 @@ const updateSchema = z.object({ tailscaleKeyCheck: z.boolean(), secretCheckTime: z.string().regex(/^\d{2}:\d{2}$/), timezone: z.string(), + integrationFailureAlerts: z.boolean(), + integrationFailureThreshold: z.number().int().min(1).max(20), }) .partial() .optional(), diff --git a/server/src/services/diagLog.ts b/server/src/services/diagLog.ts index 2bc349a..37290b6 100644 --- a/server/src/services/diagLog.ts +++ b/server/src/services/diagLog.ts @@ -1,6 +1,7 @@ import { and, desc, eq, lt, sql } from "drizzle-orm"; import { db } from "../db/client.js"; import { diagLog } from "../db/schema.js"; +import { trackIntegrationHealth } from "./integrationHealthMonitor.js"; const MAX_ENTRIES = 500; @@ -15,6 +16,7 @@ async function recordDiagEntry(entry: { source: string; operation: string; ok: b if (cutoff) { await db.delete(diagLog).where(lt(diagLog.id, cutoff.id)); } + await trackIntegrationHealth(entry.source, entry.ok); } catch (err) { console.error("[diagLog] failed to record entry:", err); } diff --git a/server/src/services/integrationHealthMonitor.ts b/server/src/services/integrationHealthMonitor.ts new file mode 100644 index 0000000..819c9e2 --- /dev/null +++ b/server/src/services/integrationHealthMonitor.ts @@ -0,0 +1,38 @@ +import { getSettings } from "./settingsStore.js"; +import { notifyIntegrationDown, notifyIntegrationRecovered } from "./notify.js"; + +interface SourceHealth { + consecutiveFailures: number; + /** True once a "down" notification has been sent for the current failure streak, so it isn't repeated on every subsequent failure. */ + alerted: boolean; +} + +/** + * In-memory only (like the diag log's own ring buffer — resets on restart, + * which is fine since this tracks a live streak, not history). Keyed by diag + * log "source" (e.g. "proxmox", "cloudflare") — the same granularity the + * Diagnostic Log itself filters by, so a homelab running two integrations of + * the same type shares one health streak between them. + */ +const health = new Map(); + +/** Called after every diagnostic-log entry is recorded, to track consecutive failures per source and alert on threshold-cross / recovery. */ +export async function trackIntegrationHealth(source: string, ok: boolean): Promise { + const state = health.get(source) ?? { consecutiveFailures: 0, alerted: false }; + + if (ok) { + if (state.alerted) { + await notifyIntegrationRecovered(source); + } + health.set(source, { consecutiveFailures: 0, alerted: false }); + return; + } + + state.consecutiveFailures += 1; + const { notifications } = await getSettings(); + if (notifications.integrationFailureAlerts && !state.alerted && state.consecutiveFailures >= notifications.integrationFailureThreshold) { + state.alerted = true; + await notifyIntegrationDown(source, state.consecutiveFailures); + } + health.set(source, state); +} diff --git a/server/src/services/notify.ts b/server/src/services/notify.ts index 04466da..66fdf02 100644 --- a/server/src/services/notify.ts +++ b/server/src/services/notify.ts @@ -191,3 +191,36 @@ export async function notifyTailscaleKeyExpiry( `${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`, ); } + +// Diagnostic-log "source" values (see integrations/*/adapter.ts and dns/adapters/*.ts) to display names. +const SOURCE_LABELS: Record = { + cloudflare: "Cloudflare", + loopia: "Loopia", + pihole: "Pi-hole", + azure: "Azure DNS", + cpanel: "cPanel", + technitium: "Technitium", + tailscale: "Tailscale", + proxmox: "Proxmox", + synology: "Synology", + semaphore: "Semaphore", + gitea: "Gitea", + dockhand: "Dockhand", +}; + +function sourceLabel(source: string): string { + return SOURCE_LABELS[source] ?? source; +} + +export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise { + if (!(await eventEnabled("integrationFailureAlerts"))) return; + await notify( + "Homelab Manager — Integration Down", + `${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`, + ); +} + +export async function notifyIntegrationRecovered(source: string): Promise { + if (!(await eventEnabled("integrationFailureAlerts"))) return; + await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`); +} diff --git a/server/src/services/settingsStore.ts b/server/src/services/settingsStore.ts index df37a96..fc325ad 100644 --- a/server/src/services/settingsStore.ts +++ b/server/src/services/settingsStore.ts @@ -42,6 +42,9 @@ export interface NotificationEvents { tailscaleKeyCheck: boolean; secretCheckTime: string; // "HH:MM" — shared by the secret-expiry and Tailscale key-expiry checks timezone: string; + integrationFailureAlerts: boolean; + /** Consecutive failed calls (from the Diagnostic Log) before an integration/DNS provider is considered down. */ + integrationFailureThreshold: number; } export type ProviderColors = Record; @@ -90,6 +93,8 @@ const DEFAULTS: AppSettings = { tailscaleKeyCheck: true, secretCheckTime: "08:00", timezone: "UTC", + integrationFailureAlerts: true, + integrationFailureThreshold: 3, }, providerColors: {}, integrationColors: {}, diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 13ef188..096033d 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -142,6 +142,8 @@ export interface NotificationEvents { tailscaleKeyCheck: boolean; secretCheckTime: string; timezone: string; + integrationFailureAlerts: boolean; + integrationFailureThreshold: number; } export type DateFormat = "ymd" | "dmy" | "mdy"; diff --git a/web/src/pages/settings/NotificationSettings.tsx b/web/src/pages/settings/NotificationSettings.tsx index 84800c8..161c3ae 100644 --- a/web/src/pages/settings/NotificationSettings.tsx +++ b/web/src/pages/settings/NotificationSettings.tsx @@ -44,6 +44,8 @@ const DEFAULT_NOTIFICATIONS: NotificationEvents = { tailscaleKeyCheck: true, secretCheckTime: "08:00", timezone: "UTC", + integrationFailureAlerts: true, + integrationFailureThreshold: 3, }; type TestResult = { ok: boolean; message: string } | null; @@ -489,6 +491,7 @@ export default function NotificationSettings() { { key: "dnsDelete" as const, label: "DNS record deleted" }, { key: "secretCheck" as const, label: "Secret expiry reminder" }, { key: "tailscaleKeyCheck" as const, label: "Tailscale key expiry reminder" }, + { key: "integrationFailureAlerts" as const, label: "Integration/DNS provider failing repeatedly" }, ].map(({ key, label }) => ( ))} +
+
+ +
+ setNotifications((n) => ({ ...n, integrationFailureThreshold: Number(e.target.value) }))} + /> + consecutive failures +
+
+ Based on the Diagnostic Log. A recovery notice is sent once it succeeds again. +
+
+
{(() => { const dailyChecksEnabled = notifications.secretCheck || notifications.tailscaleKeyCheck; return (