Alert when an integration or DNS provider fails repeatedly
The Diagnostic Log already records every outbound call's success or failure, but nothing acted on it — you'd only notice an integration was down by happening to open its page. Adds a per-source consecutive- failure counter (in-memory, reset on restart, same durability tier as the diag log's own ring buffer) hooked into recordDiagEntry: crossing the configurable threshold (default 3) sends one "down" notification on every configured channel, and a "recovered" notification fires once it succeeds again — no repeat spam while it stays down. New "Integration/DNS provider failing repeatedly" toggle and threshold field under Settings -> Notifications. Verified end-to-end against an isolated scratch database with a real local HTTP server standing in for the webhook channel: 5 consecutive failures produced exactly one "Down" notification (at the 3rd failure, correctly naming "3 calls"), a subsequent success produced exactly one "Recovered" notification, and two more failures on a fresh streak triggered nothing (below threshold) — confirmed the real dev database's mtime was untouched throughout. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
10d123b18a
commit
b5a4c6e2d9
7 files changed
+105
No files matched your search
@@ -191,3 +191,36 @@ export async function notifyTailscaleKeyExpiry(
|
||||
`${expiring.length} device key${expiring.length !== 1 ? "s" : ""} need attention:\n\n${lines.join("\n")}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Diagnostic-log "source" values (see integrations/*/adapter.ts and dns/adapters/*.ts) to display names.
|
||||
const SOURCE_LABELS: Record<string, string> = {
|
||||
cloudflare: "Cloudflare",
|
||||
loopia: "Loopia",
|
||||
pihole: "Pi-hole",
|
||||
azure: "Azure DNS",
|
||||
cpanel: "cPanel",
|
||||
technitium: "Technitium",
|
||||
tailscale: "Tailscale",
|
||||
proxmox: "Proxmox",
|
||||
synology: "Synology",
|
||||
semaphore: "Semaphore",
|
||||
gitea: "Gitea",
|
||||
dockhand: "Dockhand",
|
||||
};
|
||||
|
||||
function sourceLabel(source: string): string {
|
||||
return SOURCE_LABELS[source] ?? source;
|
||||
}
|
||||
|
||||
export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise<void> {
|
||||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||||
await notify(
|
||||
"Homelab Manager — Integration Down",
|
||||
`${sourceLabel(source)} has failed its last ${consecutiveFailures} call${consecutiveFailures !== 1 ? "s" : ""} in a row. Check the Diagnostic Log for details.`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyIntegrationRecovered(source: string): Promise<void> {
|
||||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||||
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
|
||||
}
|
||||
Reference in new issue
Block a user