From ca61f2a91535fccc7737e33f04cf12c38328029d Mon Sep 17 00:00:00 2001 From: Bobban Rydh Date: Tue, 22 Sep 2026 21:01:48 +0200 Subject: [PATCH] Surface Proxmox VMs/LXCs with no backup coverage at all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failing backup run is visible now, but a guest with no backup job covering it in the first place was still a silent gap. Rather than depending on Proxmox's /cluster/backup-info/not-backed-up-guests endpoint (only exists on newer PVE versions), this derives coverage from data already fetched: a guest counts as covered if any enabled job either lists its vmid directly, or backs up "all guests" (scoped to the job's node, if it has one) without excluding it. Adds a warning banner plus a full table to the Proxmox page's Backups card, and extends the existing daily "Proxmox backup failed" notification (relabeled to mention this too) to also list uncovered guests, gated by the same toggle. Verified the coverage logic directly (it's a pure function, so no fake server needed) across 7 cases: no jobs at all, an all-guests job with an exclude list, a specific-vmids job, a node-scoped job that shouldn't cover a guest on a different node, a disabled job providing no real coverage, two jobs whose combined scope covers everything neither would alone, and a realistic mixed scenario — all passed. Confirmed the real dev database's mtime was untouched throughout. Co-Authored-By: Claude Sonnet 5 --- server/src/integrations/proxmox/adapter.ts | 24 +++++++++++ server/src/routes/integrations.ts | 10 +++-- server/src/services/notify.ts | 12 ++++++ server/src/services/proxmoxBackupScheduler.ts | 16 ++++++-- web/src/api/client.ts | 1 + web/src/pages/Proxmox.tsx | 40 +++++++++++++++++++ .../pages/settings/NotificationSettings.tsx | 2 +- 7 files changed, 98 insertions(+), 7 deletions(-) diff --git a/server/src/integrations/proxmox/adapter.ts b/server/src/integrations/proxmox/adapter.ts index 1d4a3ca..775597e 100644 --- a/server/src/integrations/proxmox/adapter.ts +++ b/server/src/integrations/proxmox/adapter.ts @@ -129,6 +129,30 @@ export interface ProxmoxBackupTask { endTime: string | null; // ISO, null while still running } +/** + * Guests not covered by any enabled backup job — derived entirely from data + * this adapter already fetches (listGuests + listBackupJobs), rather than + * depending on Proxmox's own `/cluster/backup-info/not-backed-up-guests` + * endpoint, which only exists on newer PVE versions. A guest is "covered" by + * a job if: the job has no node restriction or matches the guest's node, and + * either the job backs up "all guests" and doesn't exclude this vmid, or the + * job explicitly lists this vmid. + */ +export function guestsWithoutBackupCoverage(guests: ProxmoxGuest[], jobs: ProxmoxBackupJob[]): ProxmoxGuest[] { + const enabledJobs = jobs.filter((j) => j.enabled); + function splitIds(csv: string | null): string[] { + return (csv ?? "").split(",").map((s) => s.trim()).filter(Boolean); + } + function isCovered(guest: ProxmoxGuest): boolean { + return enabledJobs.some((job) => { + if (job.node && job.node !== guest.node) return false; + if (job.allGuests) return !splitIds(job.exclude).includes(String(guest.vmid)); + return splitIds(job.vmids).includes(String(guest.vmid)); + }); + } + return guests.filter((g) => !isCovered(g)); +} + export interface ProxmoxAdapter { ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>; listGuests(): Promise; diff --git a/server/src/routes/integrations.ts b/server/src/routes/integrations.ts index 1a65bcb..2e2e086 100644 --- a/server/src/routes/integrations.ts +++ b/server/src/routes/integrations.ts @@ -18,7 +18,7 @@ import { createTailscaleAdapter, isKeyExpiringSoon } from "../integrations/tails import { createGiteaAdapter } from "../integrations/gitea/adapter.js"; import { createDockhandAdapter } from "../integrations/dockhand/adapter.js"; import { createSemaphoreAdapter } from "../integrations/semaphore/adapter.js"; -import { createProxmoxAdapter } from "../integrations/proxmox/adapter.js"; +import { createProxmoxAdapter, guestsWithoutBackupCoverage } from "../integrations/proxmox/adapter.js"; import { createSynologyAdapter } from "../integrations/synology/adapter.js"; import { asyncHandler } from "../utils/asyncHandler.js"; @@ -669,8 +669,12 @@ integrationsRouter.get("/:id/proxmox/backups", asyncHandler(async (req, res) => if (!found) return; try { - const [jobs, tasks] = await Promise.all([found.adapter.listBackupJobs(), found.adapter.listRecentBackupTasks()]); - res.json({ jobs, tasks }); + const [jobs, tasks, guests] = await Promise.all([ + found.adapter.listBackupJobs(), + found.adapter.listRecentBackupTasks(), + found.adapter.listGuests(), + ]); + res.json({ jobs, tasks, uncoveredGuests: guestsWithoutBackupCoverage(guests, jobs) }); } catch (err) { res.status(502).json({ error: err instanceof Error ? err.message : String(err) }); } diff --git a/server/src/services/notify.ts b/server/src/services/notify.ts index 36ea9c9..993f64d 100644 --- a/server/src/services/notify.ts +++ b/server/src/services/notify.ts @@ -257,6 +257,18 @@ export async function notifyProxmoxBackupFailure( ); } +export async function notifyProxmoxUncoveredGuests( + uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[], +): Promise { + if (uncovered.length === 0) return; + if (!(await eventEnabled("proxmoxBackupCheck"))) return; + const lines = uncovered.map((u) => `${u.guestName} (#${u.vmid}) on ${u.node} [${u.integrationName}]`); + await notify( + "Homelab Manager — Proxmox Guests Without Backup Coverage", + `${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`, + ); +} + export async function notifyTailscaleKeyExpiry( expiring: { integrationName: string; deviceLabel: string; daysLeft: number }[], ): Promise { diff --git a/server/src/services/proxmoxBackupScheduler.ts b/server/src/services/proxmoxBackupScheduler.ts index 056fe62..eab65c7 100644 --- a/server/src/services/proxmoxBackupScheduler.ts +++ b/server/src/services/proxmoxBackupScheduler.ts @@ -3,8 +3,8 @@ import { and, eq } from "drizzle-orm"; import { db } from "../db/client.js"; import { integrations } from "../db/schema.js"; import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; -import { createProxmoxAdapter, type ProxmoxBackupTask } from "../integrations/proxmox/adapter.js"; -import { notifyProxmoxBackupFailure } from "./notify.js"; +import { createProxmoxAdapter, guestsWithoutBackupCoverage, type ProxmoxBackupTask } from "../integrations/proxmox/adapter.js"; +import { notifyProxmoxBackupFailure, notifyProxmoxUncoveredGuests } from "./notify.js"; import { getSettings, getInternalFlag, setInternalFlag } from "./settingsStore.js"; const LAST_RUN_FLAG = "proxmoxBackupCheckLastRunDate"; @@ -16,13 +16,18 @@ async function checkProxmoxBackups(): Promise { .where(and(eq(integrations.type, "proxmox"), eq(integrations.enabled, true))); const failures: { integrationName: string; node: string; guestId: string | null; status: string }[] = []; + const uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[] = []; for (const row of rows) { try { const loaded = await loadIntegrationConfig(row.id); if (!loaded) continue; const adapter = createProxmoxAdapter(loaded.config as any); - const tasks = await adapter.listRecentBackupTasks(); + const [tasks, jobs, guests] = await Promise.all([ + adapter.listRecentBackupTasks(), + adapter.listBackupJobs(), + adapter.listGuests(), + ]); // Each node runs its own backup schedule independently, so check the // most recent run per node rather than only the single most recent @@ -39,12 +44,17 @@ async function checkProxmoxBackups(): Promise { failures.push({ integrationName: row.name, node, guestId: task.guestId, status: task.status }); } } + + for (const g of guestsWithoutBackupCoverage(guests, jobs)) { + uncovered.push({ integrationName: row.name, guestName: g.name, vmid: g.vmid, node: g.node }); + } } catch (err) { console.error(`[proxmoxBackup] check failed for integration ${row.id}:`, err); } } await notifyProxmoxBackupFailure(failures); + await notifyProxmoxUncoveredGuests(uncovered); } async function checkProxmoxBackupsOnce(): Promise { diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 2fdfc4c..150c6b6 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -541,6 +541,7 @@ export interface ProxmoxBackupTask { export interface ProxmoxBackupsResponse { jobs: ProxmoxBackupJob[]; tasks: ProxmoxBackupTask[]; + uncoveredGuests: ProxmoxGuest[]; } export interface SynologyVolume { diff --git a/web/src/pages/Proxmox.tsx b/web/src/pages/Proxmox.tsx index 43dcafb..eb35536 100644 --- a/web/src/pages/Proxmox.tsx +++ b/web/src/pages/Proxmox.tsx @@ -95,6 +95,7 @@ export default function Proxmox({ user }: { user: CurrentUser }) { const [backupJobs, setBackupJobs] = useState(null); const [backupTasks, setBackupTasks] = useState(null); + const [uncoveredGuests, setUncoveredGuests] = useState(null); const [backupsError, setBackupsError] = useState(null); const [loadingBackups, setLoadingBackups] = useState(false); @@ -140,6 +141,7 @@ export default function Proxmox({ user }: { user: CurrentUser }) { .then((res) => { setBackupJobs(res.jobs); setBackupTasks(res.tasks); + setUncoveredGuests(res.uncoveredGuests); }) .catch((err) => setBackupsError(err instanceof Error ? err.message : String(err))) .finally(() => setLoadingBackups(false)); @@ -155,6 +157,7 @@ export default function Proxmox({ user }: { user: CurrentUser }) { setNodeStats(null); setBackupJobs(null); setBackupTasks(null); + setUncoveredGuests(null); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [selectedId]); @@ -349,6 +352,12 @@ export default function Proxmox({ user }: { user: CurrentUser }) { {backupTasks[0].status} )} + {uncoveredGuests && uncoveredGuests.length > 0 && ( +
+ {uncoveredGuests.length} guest{uncoveredGuests.length !== 1 ? "s are" : " is"} not covered by + any enabled backup job: {uncoveredGuests.map((g) => `${g.name} (#${g.vmid})`).join(", ")} +
+ )}
Scheduled jobs
{backupJobs && backupJobs.length > 0 ? ( @@ -382,6 +391,37 @@ export default function Proxmox({ user }: { user: CurrentUser }) { !backupsError &&
No backup jobs configured in Proxmox.
)} +
Guests without backup coverage
+ {uncoveredGuests && uncoveredGuests.length > 0 ? ( + + + + + + + + + + + {uncoveredGuests.map((g) => ( + + + + + + + ))} + +
NameNodeTypeStatus
+ {g.name} #{g.vmid} + {g.node} + {g.type === "qemu" ? "VM" : "LXC"} + {proxmoxStatusBadge(g.status)}
+ ) : ( + !backupsError && + uncoveredGuests &&
Every VM/LXC is covered by at least one enabled backup job.
+ )} +
Recent runs
{backupTasks && backupTasks.length > 0 ? ( diff --git a/web/src/pages/settings/NotificationSettings.tsx b/web/src/pages/settings/NotificationSettings.tsx index a01c3ec..aeaffff 100644 --- a/web/src/pages/settings/NotificationSettings.tsx +++ b/web/src/pages/settings/NotificationSettings.tsx @@ -519,7 +519,7 @@ export default function NotificationSettings() { { key: "secretCheck" as const, label: "Secret expiry reminder" }, { key: "tailscaleKeyCheck" as const, label: "Tailscale key expiry reminder" }, { key: "dockerUpdateCheck" as const, label: "Docker image update available" }, - { key: "proxmoxBackupCheck" as const, label: "Proxmox backup failed" }, + { key: "proxmoxBackupCheck" as const, label: "Proxmox backup failed or a guest has no coverage" }, { key: "integrationFailureAlerts" as const, label: "Integration/DNS provider failing repeatedly" }, ].map(({ key, label }) => (