/** * Everything that's wrong right now, in one list — the Alerts page. Nothing here decides what counts as a problem: it asks * the same checks that send the notifications (health, backups, updates, expiry, automation, ...) and turns what they find * into a flat list, so the page and the notifications can't disagree. Unlike the notifications it ignores the per-event * on/off toggles (the page is for looking at, not for being interrupted by) and keeps problems that are under a * maintenance window, flagged as silenced rather than dropped. * * It reads live (a handful of API calls per integration), so a result is kept for a short while rather than re-run for * every viewer, and every source has a time limit so one hung integration can't hang the page. A source that can't be * read is reported as such — silence from it must never look like "all clear". */ import { eq } from "drizzle-orm"; import { db } from "../db/client.js"; import { endpointChecks, haEntities, haWatched, heartbeats, integrations, networkDevices, secrets, servers } from "../db/schema.js"; import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js"; import { storageFillingSoon, storageOverview } from "./storageHistory.js"; import { daysOfSupportLeft, lifecycleOf } from "./osLifecycle.js"; import { npmIntegrationIds, npmProblems, readNpm } from "./npmReader.js"; import { haProblems, haSyncStatuses, isHaFresh } from "./homeAssistantSync.js"; import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js"; import { getBackupCoverage, staleBackups } from "./backupCoverage.js"; import { recordFromReport } from "./incidents.js"; import { reconcileAcks } from "./alertAcks.js"; import { flappingKeys, keyOfAlertId } from "./flapGuard.js"; import { loadSnoozes } from "./alertSnoozes.js"; import { lastAttempt as lastSelfBackup, newestSnapshot, STALE_AFTER_HOURS as SELF_BACKUP_STALE_HOURS } from "./selfBackup.js"; import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js"; import { createOsTicketAdapter } from "../integrations/osticket/adapter.js"; import { activeSubjects, isSourceInMaintenance, subjectOfConditionKey } from "./maintenance.js"; import { collectSnapshot, evaluateHealth, startupGraceRemainingMs } from "./healthMonitor.js"; import { collectAutomation, evaluateAutomation } from "./automationMonitor.js"; import { collectDockerUpdates } from "./dockerUpdateScheduler.js"; import { collectProxmoxBackupProblems } from "./proxmoxBackupScheduler.js"; import { collectPbsProblems } from "./pbsVerificationScheduler.js"; import { collectTailscaleKeyExpiries } from "./tailscaleKeyExpiryScheduler.js"; import { collectDomainAlerts } from "./domainMonitor.js"; import { computeSecretStatus } from "./secretStatus.js"; import { getFailingSources } from "./integrationHealthMonitor.js"; import { getSettings } from "./settingsStore.js"; import { sourceLabel } from "./notify.js"; import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js"; import { endpointConditions } from "./endpointChecks.js"; import { heartbeatConditions } from "./heartbeats.js"; import { latestTests, restoreAlerts } from "./restoreTests.js"; import { kindOfKey, type Alert, type AlertCategory, type AlertSeverity, type SourceFailure } from "./alertTypes.js"; export interface AlertsReport { alerts: Alert[]; /** Active problems by severity — those under a maintenance window are counted apart, not in these. */ counts: { critical: number; warning: number; info: number; silenced: number; /** Held back because another problem explains them (a guest down because its host is). */ grouped: number }; /** Things that couldn't be checked, and which checks that leaves blind. */ couldntCheck: { name: string; error: string; affects: string[] }[]; /** Context worth knowing about how complete the picture is. */ notes: string[]; generatedAt: string; } /** Where each kind of source lives in the app, and what to call it. */ const SOURCES: Record = { server: { label: "Server", link: "/servers" }, proxmox: { label: "Proxmox", link: "/proxmox" }, synology: { label: "Synology", link: "/synology" }, semaphore: { label: "Semaphore", link: "/semaphore" }, gitea: { label: "Gitea", link: "/gitea" }, dockhand: { label: "Docker", link: "/docker" }, tailscale: { label: "Tailscale", link: "/tailscale" }, homelab: { label: "Homelab Manager", link: "/settings/data" }, pbs: { label: "Proxmox Backup", link: "/backups?view=pbs" }, uptimekuma: { label: "Uptime Kuma", link: "/uptime-kuma" }, osticket: { label: "osTicket", link: "/osticket" }, unifi: { label: "UniFi", link: "/network/devices" }, npm: { label: "Nginx Proxy Manager", link: "/proxy-hosts" }, homeassistant: { label: "Home Assistant", link: "/home-assistant" }, ups: { label: "UPS", link: "/ups" }, slo: { label: "Service level", link: "/alert-settings?view=levels" }, duplicati: { label: "Duplicati", link: "/duplicati" }, jellyfin: { label: "Jellyfin", link: "/jellyfin" }, authentik: { label: "Authentik", link: "/authentik" }, prometheus: { label: "Prometheus", link: "/prometheus" }, immich: { label: "Immich", link: "/immich" }, paperless: { label: "Paperless-ngx", link: "/paperless" }, n8n: { label: "n8n", link: "/n8n" }, netbox: { label: "NetBox", link: "/netbox" }, nextcloud: { label: "Nextcloud", link: "/nextcloud" }, hetzner: { label: "Hetzner", link: "/hetzner" }, speedtest: { label: "Speedtest", link: "/speedtest" }, fragnet: { label: "Fragnet", link: "/game-servers" }, sonarr: { label: "Sonarr", link: "/media" }, radarr: { label: "Radarr", link: "/media" }, lidarr: { label: "Lidarr", link: "/media" }, readarr: { label: "Readarr", link: "/media" }, prowlarr: { label: "Prowlarr", link: "/media" }, hetznerrobot: { label: "Hetzner Robot", link: "/hetzner-robot" }, qbittorrent: { label: "qBittorrent", link: "/downloads" }, sabnzbd: { label: "SABnzbd", link: "/downloads" }, adguard: { label: "AdGuard Home", link: "/adguard" }, technitium: { label: "Technitium DNS", link: "/technitium" }, versions: { label: "Versions", link: "/versions" }, grafana: { label: "Grafana", link: "/grafana" }, secrets: { label: "Secrets", link: "/secrets" }, domains: { label: "Domains", link: "/dns?view=domains" }, endpoint: { label: "Endpoint", link: "/endpoints" }, heartbeat: { label: "Heartbeat", link: "/heartbeats" }, }; const SOURCE_TIMEOUT_MS = 20_000; /** How long a result is reused. */ const CACHE_MS = 60_000; /** Pressing Refresh over and over shouldn't hammer every integration — a result younger than this is reused even then. */ const MIN_REFRESH_MS = 10_000; const SEVERITY_ORDER: Record = { critical: 0, warning: 1, info: 2 }; /** Which kind of source, and which one, a health/automation condition key belongs to. */ function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } { const device = /^offline:device:(\d+)$/.exec(key); if (device) return { type: "unifi", id: Number(device[1]), category: "offline" }; let m = /^rule:\d+:server:(\d+)/.exec(key); if (m) return { type: "server", id: Number(m[1]), category: "monitoring" }; m = /^baseline:(?:cpu|mem):server:(\d+)$/.exec(key); if (m) return { type: "server", id: Number(m[1]), category: "monitoring" }; m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key); if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : key.startsWith("new-port") ? "monitoring" : "disk" }; m = /^disk:proxmox:(\d+):/.exec(key); if (m) return { type: "proxmox", id: Number(m[1]), category: "disk" }; m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key); if (m) return { type: "synology", id: Number(m[1]), category: "disk" }; m = /^automation:(semaphore|gitea):(\d+):/.exec(key); if (m) return { type: m[1], id: Number(m[2]), category: "automation" }; m = /^ups-(?:battery|low|replace|load):(\d+)$/.exec(key); if (m) return { type: "ups", id: Number(m[1]), category: "monitoring" }; m = /^slo:(\d+)$/.exec(key); if (m) return { type: "slo", id: Number(m[1]), category: "monitoring" }; m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key); if (m) return { type: "dockhand", id: Number(m[1]), category: "services" }; m = /^docker-(?:vuln(?:-high)?|image-age):(\d+):/.exec(key); if (m) return { type: "dockhand", id: Number(m[1]), category: "services" }; m = /^speedtest-(?:slow|failing):(\d+)/.exec(key); if (m) return { type: "speedtest", id: Number(m[1]), category: "monitoring" }; m = /^hetzner-(off|stuck|rescue|traffic|forecast|backup|lb|cert|fip):(\d+):/.exec(key); if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" || m[1] === "forecast" ? "monitoring" : m[1] === "backup" ? "backup" : "services" }; m = /^version-behind:(\d+)$/.exec(key); if (m) return { type: "versions", id: Number(m[1]), category: "updates" }; m = /^adguard-(stopped|dns|protection|filters):(\d+)/.exec(key); if (m) return { type: "adguard", id: Number(m[2]), category: m[1] === "stopped" || m[1] === "dns" ? "services" : "monitoring" }; m = /^technitium-(dns|blocking|lists|zones|node|config):(\d+)/.exec(key); if (m) return { type: "technitium", id: Number(m[2]), category: m[1] === "dns" || m[1] === "zones" || m[1] === "node" ? "services" : "monitoring" }; m = /^grafana-(firing|rule-error|ds):(\d+):/.exec(key); if (m) return { type: "grafana", id: Number(m[2]), category: m[1] === "ds" ? "integration" : "monitoring" }; m = /^(sonarr|radarr|prowlarr|lidarr|readarr)-(health|queue|disk):(\d+)/.exec(key); if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" }; m = /^hetznerrobot-(ending|rescue|setup|boxlocked|boxending):(\d+):/.exec(key); if (m) return { type: "hetznerrobot", id: Number(m[2]), category: "services" }; m = /^(qbittorrent|sabnzbd)-(stuck|connection|paused|stalled|disk):(\d+)/.exec(key); if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" }; m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key); if (m) return { type: "fragnet", id: Number(m[2]), category: m[1] === "disk" ? "disk" : m[1] === "backup" ? "backup" : "monitoring" }; m = /^duplicati-(?:failed|stale|warnings|streak|growth):(\d+)/.exec(key); if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" }; m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key); if (m) return { type: "nextcloud", id: Number(m[1]), category: key.startsWith("nextcloud-disk") ? "disk" : "services" }; m = /^netbox-(?:workers):(\d+)/.exec(key); if (m) return { type: "netbox", id: Number(m[1]), category: "services" }; m = /^n8n-(?:failed):(\d+)/.exec(key); if (m) return { type: "n8n", id: Number(m[1]), category: "automation" }; m = /^paperless-(?:component|failed|disk):(\d+)/.exec(key); if (m) return { type: "paperless", id: Number(m[1]), category: key.startsWith("paperless-disk") ? "disk" : "services" }; m = /^immich-(?:disk|jobs):(\d+)/.exec(key); if (m) return { type: "immich", id: Number(m[1]), category: key.startsWith("immich-disk") ? "disk" : "services" }; m = /^prometheus-(?:target|alert):(\d+)/.exec(key); if (m) return { type: "prometheus", id: Number(m[1]), category: "monitoring" }; m = /^authentik-(?:outpost|outpost-version|cert|token|worker|logins|errors):(\d+)/.exec(key); if (m) return { type: "authentik", id: Number(m[1]), category: /^authentik-(?:cert|token):/.test(key) ? "expiry" : /^authentik-(?:outpost|worker)/.test(key) ? "services" : "monitoring" }; m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key); if (m) return { type: "jellyfin", id: Number(m[1]), category: key.startsWith("jellyfin-task") ? "automation" : "monitoring" }; return { type: "server", id: null, category: "disk" }; } function withTimeout(work: Promise): Promise { let timer: ReturnType; const limit = new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`timed out after ${SOURCE_TIMEOUT_MS / 1000}s`)), SOURCE_TIMEOUT_MS); }); return Promise.race([work, limit]).finally(() => clearTimeout(timer)); } const errorText = (err: unknown) => (err instanceof Error ? err.message : String(err)); const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`; export async function collectAlerts(): Promise { const alerts: Alert[] = []; const notes: string[] = []; const blind = new Map }>(); const subjects = await activeSubjects(); const intRows = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations); const intById = new Map(intRows.map((r) => [r.id, r])); const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((r) => [r.id, r.name])); const deviceNames = new Map((await db.select({ id: networkDevices.id, name: networkDevices.name }).from(networkDevices)).map((r) => [r.id, r.name])); /** The thing a maintenance window would go on to silence an alert about `subject` ("server:3", "integration:1"). */ const targetOf = (type: "server" | "integration" | "dns_provider" | "device", id: number): Alert["target"] => ({ type, id, name: (type === "server" ? serverNames.get(id) : type === "device" ? deviceNames.get(id) : type === "integration" ? intById.get(id)?.name : undefined) ?? `${type} #${id}`, }); const targetOfSubject = (subject: string | null): Alert["target"] => { const m = /^(server|integration|dns_provider|device):(\d+)$/.exec(subject ?? ""); return m ? targetOf(m[1] as "server" | "integration" | "dns_provider" | "device", Number(m[2])) : null; }; const integrationTarget = (id: number): Alert["target"] => targetOf("integration", id); /** Integration-down alerts are per kind of service; a window on any integration of that kind silences them, so offer the first. */ const firstIntegrationOfType = (type: string): Alert["target"] => { const row = intRows.find((r) => r.type === type && r.enabled); return row ? integrationTarget(row.id) : null; }; function push(a: { category: AlertCategory; severity: AlertSeverity; type: string; message: string; key: string; link?: string | null; silenced?: boolean; target?: Alert["target"]; causedBy?: Alert["causedBy"]; affects?: string[] }) { const meta = SOURCES[a.type]; alerts.push({ id: `${a.category}:${a.key}`, kind: kindOfKey(a.key), type: a.type, severity: a.severity, category: a.category, source: meta?.label ?? sourceLabel(a.type), message: a.message, link: a.link === undefined ? (meta?.link ?? null) : a.link, silenced: !!a.silenced || !!a.causedBy, target: a.target ?? null, ...(a.causedBy ? { causedBy: a.causedBy } : {}), ...(a.affects?.length ? { affects: a.affects } : {}), }); } function cannotCheck(name: string, error: string, check: string) { const entry = blind.get(name) ?? { error, affects: new Set() }; entry.affects.add(check); blind.set(name, entry); } const cannotCheckIntegration = (f: SourceFailure, check: string) => { const row = intById.get(f.integrationId); cannotCheck(`${row ? (SOURCES[row.type]?.label ?? row.type) : "Integration"} “${f.integrationName}”`, f.message, check); }; const silencedIntegration = (id: number) => subjects.has(`integration:${id}`); // One entry per check. A check that blows up, or hangs, only costs its own section of the page. async function check(name: string, work: () => Promise) { try { await withTimeout(work()); } catch (err) { cannotCheck(name, errorText(err), name); } } // The servers found to be offline, for the integration failures that one may explain: the server check fills it in (and it is empty if that check fails). let offlineKnown!: (ids: Set) => void; const offlineServers = new Promise>((resolve) => (offlineKnown = resolve)); // The same for network devices, which can explain a service that has stopped answering. let devicesKnown!: (ids: Set) => void; const offlineDeviceIds = new Promise>((resolve) => (devicesKnown = resolve)); await Promise.all([ check("Server and storage health", async () => { const { healthChecks } = await getSettings(); const { snapshot, held } = await collectSnapshot(); const now = Date.now(); const grace = startupGraceRemainingMs(now); if (grace > 0) { notes.push( `Server-offline and server-disk checks are paused for another ${Math.ceil(grace / 60_000)} min after the app restarted, so agents get a chance to report before any server is judged.`, ); } const conditions = evaluateHealth(snapshot, healthChecks, now, { skipServers: grace > 0 }); // A server that is down because the one it runs on is down is folded under that one. const offlineIds = new Set(); const offlineDevices = new Set(); for (const c of conditions) { const m = /^offline:(server|device):(\d+)$/.exec(c.key); if (m) (m[1] === "server" ? offlineIds : offlineDevices).add(Number(m[2])); } // (Or because of a service it needs that is failing, or a server it needs that needs one.) offlineKnown(offlineIds); devicesKnown(offlineDevices); const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source), offlineDevices); for (const c of conditions) { const origin = originOfConditionKey(c.key); const subject = subjectOfConditionKey(c.key); const offlineMatch = /^offline:(server|device):(\d+)$/.exec(c.key); const offlineNode = offlineMatch ? `${offlineMatch[1]}:${offlineMatch[2]}` : undefined; const parentOf = offlineNode ? cover.cause.get(offlineNode) : undefined; const upsOnBattery = /^ups-battery:(\d+)$/.exec(c.key); push({ causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf, alertId: alertIdOfNode(parentOf, cover.graph) ?? undefined } : undefined, affects: offlineNode ? cover.taken.get(offlineNode)?.sort((a, b) => a.localeCompare(b)) : upsOnBattery ? cover.taken.get(`integration:${upsOnBattery[1]}`)?.sort((a, b) => a.localeCompare(b)) : undefined, category: origin.category, severity: c.severity ?? "warning", type: origin.type, message: c.message, key: c.key, link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : origin.type === "sonarr" || origin.type === "radarr" || origin.type === "prowlarr" || origin.type === "lidarr" || origin.type === "readarr" ? "/media" : origin.type === "adguard" ? "/adguard" : origin.type === "grafana" ? "/grafana" : undefined, silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject), }); } // Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.) if (held.has("server:rules")) cannotCheck("Alert rules", "couldn't be judged — see the server log", "Server and storage health"); if (held.has("duplicati")) cannotCheck("Duplicati backups", "couldn't be judged — see the server log", "Server and storage health"); if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health"); if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health"); for (const h of held) { const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|hetznerrobot|speedtest|fragnet|sonarr|radarr|prowlarr|lidarr|readarr|qbittorrent|sabnzbd|adguard|technitium|grafana):(\d+)(?::(.+))?$/.exec(h); if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue; const row = intById.get(Number(m[2])); cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] === "images" ? " (age of the images)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health"); } }).finally(() => { offlineKnown(new Set()); devicesKnown(new Set()); }), check("Automation runs", async () => { const { items, held } = await collectAutomation(); for (const c of evaluateAutomation(items).conditions) { const origin = originOfConditionKey(c.key); const subject = subjectOfConditionKey(c.key); push({ category: "automation", severity: "warning", type: origin.type, message: c.message, key: c.key, silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject) }); } for (const h of held) { const m = /^(semaphore|gitea):(\d+)$/.exec(h); if (!m) continue; const row = intById.get(Number(m[2])); cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”`, "couldn't be read — see the Diagnostic Log", "Automation runs"); } }), check("Proxmox backups", async () => { const { failures, uncovered, sourceFailures } = await collectProxmoxBackupProblems({ skipSilenced: false }); for (const f of failures) { push({ category: "backup", severity: "critical", type: "proxmox", message: `Latest backup on ${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} didn't succeed [${f.integrationName}]: ${f.status}`, key: `proxmox-backup:${f.integrationId}:${f.node}`, silenced: f.silenced, target: integrationTarget(f.integrationId), }); } for (const u of uncovered) { push({ category: "backup", severity: "warning", type: "proxmox", message: `${u.guestName} (#${u.vmid}) on ${u.node} isn't covered by any backup job [${u.integrationName}]`, key: `proxmox-uncovered:${u.integrationId}:${u.vmid}`, silenced: u.silenced, target: integrationTarget(u.integrationId), }); } sourceFailures.forEach((f) => cannotCheckIntegration(f, "Proxmox backups")); }), // A guest whose newest backup is older than its limit — which Proxmox's own "latest run failed" alert can't see, because that only // looks at the most recent run per node. check("Backup coverage", async () => { const now = new Date(); const report = await getBackupCoverage({ now, rules: true }); for (const s of staleBackups(report, now, subjects)) { push({ category: "backup", severity: s.severity, type: "proxmox", message: s.message, key: s.key, link: "/backups", silenced: s.silenced, target: s.guest.server ? targetOf("server", s.guest.server.id) : integrationTarget(s.guest.integrationId), }); } // A restore that failed, or hasn't been tried in too long (the limit is in Settings; a failure is always raised). const { healthChecks } = await getSettings(); for (const r of restoreAlerts(report, await latestTests(), healthChecks.restoreTestDays, now)) { const g = r.guest; push({ category: "backup", severity: r.severity, type: g ? "proxmox" : "homelab", message: r.message, key: r.key, link: "/backups", silenced: g ? subjects.has(`integration:${g.integrationId}`) || (g.server !== null && subjects.has(`server:${g.server.id}`)) : false, target: g ? (g.server ? targetOf("server", g.server.id) : integrationTarget(g.integrationId)) : null, }); } }), // What the endpoint checks have found: an address, port or certificate that has stopped being fine. check("Endpoint checks", async () => { for (const c of endpointConditions(await db.select().from(endpointChecks), new Date())) { push({ category: c.category, severity: c.severity, type: "endpoint", message: c.message, key: c.key, link: "/endpoints", silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`), target: c.serverId !== null ? targetOf("server", c.serverId) : null, }); } }), // Jobs that report in: one that has not done so in time, or said it failed. check("Heartbeats", async () => { for (const c of heartbeatConditions(await db.select().from(heartbeats), new Date())) { push({ category: c.category, severity: c.severity, type: "heartbeat", message: c.message, key: c.key, link: "/heartbeats", silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`), target: c.serverId !== null ? targetOf("server", c.serverId) : null, }); } }), // Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly // stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it. check("Homelab Manager's own backups", async () => { const { selfBackup } = await getSettings(); if (!selfBackup.enabled) return; const newest = newestSnapshot(); const last = await lastSelfBackup(); const failing = last && !last.ok ? ` The last attempt failed: ${last.error}` : ""; if (!newest) { if (failing) push({ category: "backup", severity: "warning", type: "homelab", message: `Homelab Manager has no backup of its own yet.${failing}`, key: "self-backup" }); return; } const ageHours = (Date.now() - Date.parse(newest.at)) / 3600_000; if (ageHours <= SELF_BACKUP_STALE_HOURS) return; push({ category: "backup", severity: ageHours >= 24 * 7 ? "critical" : "warning", type: "homelab", message: `The newest backup of Homelab Manager's own database is ${ageHours < 72 ? `${Math.round(ageHours)} h` : `${Math.floor(ageHours / 24)} days`} old.${failing}`, key: "self-backup", }); }), check("Backup verification", async () => { const { problems, sourceFailures } = await collectPbsProblems({ skipSilenced: false }); for (const p of problems) { push({ category: "backup", severity: p.error ? "warning" : "critical", type: "pbs", message: p.error ? `Datastore "${p.datastore}" couldn't be read [${p.integrationName}]: ${p.error}` : `Datastore "${p.datastore}" has ${plural(p.failedCount, "snapshot")} that failed verification [${p.integrationName}]`, key: `pbs:${p.integrationId}:${p.datastore}`, silenced: p.silenced, target: integrationTarget(p.integrationId), }); } sourceFailures.forEach((f) => cannotCheckIntegration(f, "Backup verification")); }), check("Image updates", async () => { const { items, failures } = await collectDockerUpdates(); for (const u of items) { push({ category: "updates", severity: "info", type: "dockhand", message: `${u.containerName} [${u.environmentName}, ${u.integrationName}] has an image update available${u.newerVersion ? ` → ${u.newerVersion}` : ""}`, key: `docker:${u.integrationId}:${u.environmentName}:${u.containerName}`, silenced: silencedIntegration(u.integrationId), target: integrationTarget(u.integrationId), }); } failures.forEach((f) => cannotCheckIntegration(f, "Image updates")); }), check("Tailscale keys", async () => { const { items, failures } = await collectTailscaleKeyExpiries(); for (const k of items) { push({ category: "expiry", severity: k.daysLeft < 0 ? "critical" : "warning", type: "tailscale", message: k.daysLeft < 0 ? `Key for ${k.deviceLabel} [${k.integrationName}] has expired` : `Key for ${k.deviceLabel} [${k.integrationName}] expires in ${plural(k.daysLeft, "day")}`, key: `tailscale-key:${k.integrationId}:${k.deviceLabel}`, silenced: silencedIntegration(k.integrationId), target: integrationTarget(k.integrationId), }); } failures.forEach((f) => cannotCheckIntegration(f, "Tailscale keys")); }), check("Uptime Kuma", async () => { for (const row of intRows.filter((r) => r.type === "uptimekuma" && r.enabled)) { try { const loaded = await loadIntegrationConfig(row.id); if (!loaded) continue; for (const m of await createUptimeKumaAdapter(loaded.config as any).listMonitors()) { if (m.status !== "down") continue; push({ category: "monitoring", severity: "critical", type: "uptimekuma", message: `Monitor "${m.name}" is down${m.target ? ` (${m.target}${m.port ? `:${m.port}` : ""})` : ""} [${row.name}]`, key: `kuma:${row.id}:${m.id}`, silenced: silencedIntegration(row.id), target: integrationTarget(row.id), }); } } catch (err) { cannotCheckIntegration({ integrationId: row.id, integrationName: row.name, message: errorText(err) }, "Uptime Kuma"); } } }), check("osTicket", async () => { for (const row of intRows.filter((r) => r.type === "osticket" && r.enabled)) { try { const loaded = await loadIntegrationConfig(row.id); if (!loaded) continue; const overdue = (await createOsTicketAdapter(loaded.config as any).listOpenTickets()).filter((t) => t.isOverdue).length; if (overdue > 0) { push({ category: "tickets", severity: "warning", type: "osticket", message: `${plural(overdue, "open ticket")} ${overdue === 1 ? "is" : "are"} overdue [${row.name}]`, key: `osticket:${row.id}`, silenced: silencedIntegration(row.id), target: integrationTarget(row.id), }); } } catch (err) { cannotCheckIntegration({ integrationId: row.id, integrationName: row.name, message: errorText(err) }, "osTicket"); } } }), check("Secrets", async () => { for (const s of await db.select().from(secrets)) { const status = computeSecretStatus(s.expiryDate, s.warnDays); if (status.status === "expired") { push({ category: "expiry", severity: "critical", type: "secrets", message: `Secret "${s.name}" expired ${plural(Math.abs(status.daysLeft), "day")} ago (${s.expiryDate})`, key: `secret:${s.id}` }); } else if (status.status === "expiring") { push({ category: "expiry", severity: "warning", type: "secrets", message: `Secret "${s.name}" expires in ${plural(status.daysLeft, "day")} (${s.expiryDate})`, key: `secret:${s.id}` }); } if (s.checkHost && s.lastCheckError) { push({ category: "expiry", severity: "warning", type: "secrets", message: `Couldn't read the live certificate for "${s.name}" (${s.checkHost}:${s.checkPort ?? 443}) — the expiry shown may be stale: ${s.lastCheckError}`, key: `secret-check:${s.id}`, }); } } }), check("Domains", async () => { const { expiring, staleChecks } = await collectDomainAlerts(); for (const d of expiring) { push({ category: "expiry", severity: d.status === "expired" ? "critical" : "warning", type: "domains", message: d.status === "expired" ? `Domain ${d.name} expired on ${d.expiresAt}` : `Domain ${d.name} expires in ${plural(d.daysLeft, "day")} (${d.expiresAt})`, key: `domain:${d.name}`, }); } for (const s of staleChecks) { push({ category: "expiry", severity: "info", type: "domains", message: `Couldn't refresh the registration for ${s.name} — the expiry shown may be stale: ${s.error}`, key: `domain-stale:${s.name}` }); } }), // Not a notification: a disk that is on course to fill is something to see coming, so it is shown here, from the server's own history. check("Disk forecasts", async () => { const { healthChecks } = await getSettings(); for (const s of await db.select({ id: servers.id, name: servers.name, thresholds: servers.thresholds }).from(servers)) { // A server may have its own forecast window (0 turns it off for that server), its own limit, and disks it is told to leave alone. const own = parseThresholds(s.thresholds); const forecastDays = effectiveThresholds(healthChecks, own).diskForecastDays; if (!(forecastDays > 0)) continue; const series = await getForecastSeries(s.id); for (const f of disksFillingSoon(series, forecastDays, (mount) => diskLimit(own, mount, healthChecks.diskUsagePercent))) { push({ category: "disk", severity: f.daysToFull <= 3 ? "critical" : "warning", type: "server", message: `Disk ${f.mount} on ${s.name} is filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")} (${Math.round(f.latest)}% used now)`, key: `disk-forecast:${s.id}:${f.mount}`, link: `/servers/${s.id}`, silenced: subjects.has(`server:${s.id}`), target: targetOf("server", s.id), }); } } }), // The same for the pools the integrations report (a Proxmox storage, a Synology volume), from the history kept on each health pass. check("Storage forecasts", async () => { const { healthChecks } = await getSettings(); if (!(healthChecks.diskForecastDays > 0)) return; const nowMs = Date.now(); const rows = (await storageOverview()).filter((r) => nowMs - Date.parse(r.updatedAt) < 2 * 86_400_000 && intById.get(r.integrationId)?.enabled); for (const f of storageFillingSoon(rows, healthChecks.diskForecastDays, healthChecks.diskUsagePercent)) { const row = intById.get(f.integrationId)!; push({ category: "disk", severity: f.daysToFull <= 3 ? "critical" : "warning", type: row.type, message: `${f.label} is filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")} (${Math.round(f.latest)}% used now)`, key: `storage-forecast:${f.integrationId}:${f.key}`, silenced: silencedIntegration(f.integrationId), target: integrationTarget(f.integrationId), }); } }), // Certificates in use by Nginx Proxy Manager that are running out, and hosts it could not set up. check("Nginx Proxy Manager", async () => { for (const i of await npmIntegrationIds()) { const read = await readNpm(i.id); if (!read.snapshot) { cannotCheck(`Nginx Proxy Manager “${i.name}”`, read.error ?? "couldn't be read", "Nginx Proxy Manager"); continue; } for (const p of npmProblems(i.id, i.name, read.snapshot, new Date())) { push({ category: p.kind === "certificate" ? "expiry" : "monitoring", severity: p.severity, type: "npm", message: p.message, key: p.key, link: "/proxy-hosts", silenced: silencedIntegration(i.id), target: integrationTarget(i.id) }); } } }), // What was chosen to watch in Home Assistant: an entity that has said nothing for a while, or a number past its limit. check("Home Assistant", async () => { const statuses = await haSyncStatuses(); const nowMs = Date.now(); const watched = await db.select().from(haWatched); if (watched.length === 0) return; const entities = await db.select().from(haEntities); for (const i of intRows.filter((r) => r.type === "homeassistant" && r.enabled)) { const mine = watched.filter((w) => w.integrationId === i.id); if (mine.length === 0) continue; if (!isHaFresh(statuses[i.id], nowMs)) { cannotCheck(`Home Assistant “${i.name}”`, statuses[i.id]?.error ?? "hasn't been read yet", "Home Assistant"); continue; } const rows = new Map(entities.filter((e) => e.integrationId === i.id).map((e) => [e.entityId, e])); for (const p of haProblems(i.id, i.name, mine, rows, new Date(nowMs))) { push({ category: "monitoring", severity: p.severity, type: "homeassistant", message: p.message, key: p.key, link: "/home-assistant", silenced: silencedIntegration(i.id), target: integrationTarget(i.id) }); } } }), // Not a notification either: an operating system whose support has ended (or is about to) is seen here, with the date, from what the agent says it runs. check("Operating system support", async () => { const { healthChecks } = await getSettings(); if (!(healthChecks.osSupportWarnDays > 0)) return; const now = new Date(); for (const s of await db.select({ id: servers.id, name: servers.name, osId: servers.osId, osName: servers.osName, osVersion: servers.osVersion }).from(servers)) { if (!s.osId) continue; const life = lifecycleOf({ id: s.osId, name: s.osName, version: s.osVersion }); const left = life ? daysOfSupportLeft(life.endsOn, now) : null; if (!life || life.endsOn === null || left === null || left > healthChecks.osSupportWarnDays) continue; const ended = left < 0; push({ category: "updates", severity: ended ? "warning" : "info", type: "server", message: ended ? `${s.name} runs ${life.product}, whose support ended on ${life.endsOn} — it no longer gets security fixes` : `${s.name} runs ${life.product}, whose support ends on ${life.endsOn} (in ${plural(left, "day")})`, key: `os-support:${s.id}`, link: `/servers/${s.id}`, silenced: subjects.has(`server:${s.id}`), target: targetOf("server", s.id), }); } }), check("Integration failures", async () => { const { notifications } = await getSettings(); const failing = getFailingSources(); // A failure explained by something it needs that is down is folded under that, like a guest under its host. const cover = failing.length > 0 ? await dependencyCover(await offlineServers, failing.map((f) => f.source), await offlineDeviceIds).catch(() => null) : null; for (const f of failing) { const cause = cover ? integrationTypeCause(f.source, cover) : null; const affects = cover ? takenByType(f.source, cover) : []; push({ causedBy: cause ? { name: cause.name, alertId: (cover && alertIdOfNode(cause.key, cover.graph)) || undefined } : undefined, affects: affects.length > 0 ? affects.sort((a, b) => a.localeCompare(b)) : undefined, category: "integration", severity: f.alerted ? "critical" : "warning", type: f.source, message: `${sourceLabel(f.source)} has failed its last ${plural(f.consecutiveFailures, "call")} in a row${f.alerted ? "" : ` (a notification goes out after ${notifications.integrationFailureThreshold})`} — see the Diagnostic Log`, key: `failing:${f.source}`, link: null, silenced: await isSourceInMaintenance(f.source), target: firstIntegrationOfType(f.source), }); } }), ]); alerts.sort( (a, b) => Number(a.silenced) - Number(b.silenced) || SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity] || a.source.localeCompare(b.source) || a.message.localeCompare(b.message), ); // Those that keep coming and going are marked, so nobody mistakes the quiet of their held messages for the problem having gone. const flapping = await flappingKeys().catch(() => new Set()); if (flapping.size > 0) for (const a of alerts) if (flapping.has(keyOfAlertId(a.id, a.category))) a.flapping = true; // Those somebody has snoozed are held back like those under maintenance, with when they come back. const snoozes = await loadSnoozes().catch(() => new Map()); if (snoozes.size > 0) { for (const a of alerts) { const sn = snoozes.get(a.id); if (sn) { if (sn.closed) a.closed = { by: sn.snoozedBy, at: sn.snoozedAt, note: sn.note }; else a.snoozed = { until: sn.until, by: sn.snoozedBy }; a.silenced = true; } } } const active = alerts.filter((a) => !a.silenced); const grouped = alerts.filter((a) => a.causedBy).length; return { alerts, counts: { critical: active.filter((a) => a.severity === "critical").length, warning: active.filter((a) => a.severity === "warning").length, info: active.filter((a) => a.severity === "info").length, silenced: alerts.length - active.length - grouped, grouped, }, couldntCheck: [...blind.entries()].map(([name, v]) => ({ name, error: v.error, affects: [...v.affects] })), notes, generatedAt: new Date().toISOString(), }; } let cache: { at: number; report: AlertsReport } | null = null; /** Forget the stored result, so the next look is a fresh one — after something changes what it would say (a maintenance window starting or ending). */ export function invalidateAlertsCache(): void { cache = null; } let inFlight: Promise | null = null; /** The current alerts, reusing a recent result unless `force` asks for a fresh one (and even then not more than once every few seconds). */ export async function getAlerts(force: boolean): Promise { const age = cache ? Date.now() - cache.at : Infinity; if (cache && age < (force ? MIN_REFRESH_MS : CACHE_MS)) return { ...cache.report, cached: true }; inFlight ??= collectAlerts() .then((report) => { cache = { at: Date.now(), report }; // Remembered for the incident record and the acknowledgements — in the background, and never allowed to hold up or break the answer. void recordFromReport(report).then(() => reconcileAcks(new Set(report.alerts.map((a) => a.id)))).catch((err) => console.error("[alerts] couldn't update the incident record:", err)); return report; }) .finally(() => { inFlight = null; }); return { ...(await inFlight), cached: false }; }