Surface Proxmox VMs/LXCs with no backup coverage at all
A failing backup run is visible now, but a guest with no backup job covering it in the first place was still a silent gap. Rather than depending on Proxmox's /cluster/backup-info/not-backed-up-guests endpoint (only exists on newer PVE versions), this derives coverage from data already fetched: a guest counts as covered if any enabled job either lists its vmid directly, or backs up "all guests" (scoped to the job's node, if it has one) without excluding it. Adds a warning banner plus a full table to the Proxmox page's Backups card, and extends the existing daily "Proxmox backup failed" notification (relabeled to mention this too) to also list uncovered guests, gated by the same toggle. Verified the coverage logic directly (it's a pure function, so no fake server needed) across 7 cases: no jobs at all, an all-guests job with an exclude list, a specific-vmids job, a node-scoped job that shouldn't cover a guest on a different node, a disabled job providing no real coverage, two jobs whose combined scope covers everything neither would alone, and a realistic mixed scenario — all passed. Confirmed the real dev database's mtime was untouched throughout. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -129,6 +129,30 @@ export interface ProxmoxBackupTask {
|
||||
endTime: string | null; // ISO, null while still running
|
||||
}
|
||||
|
||||
/**
|
||||
* Guests not covered by any enabled backup job — derived entirely from data
|
||||
* this adapter already fetches (listGuests + listBackupJobs), rather than
|
||||
* depending on Proxmox's own `/cluster/backup-info/not-backed-up-guests`
|
||||
* endpoint, which only exists on newer PVE versions. A guest is "covered" by
|
||||
* a job if: the job has no node restriction or matches the guest's node, and
|
||||
* either the job backs up "all guests" and doesn't exclude this vmid, or the
|
||||
* job explicitly lists this vmid.
|
||||
*/
|
||||
export function guestsWithoutBackupCoverage(guests: ProxmoxGuest[], jobs: ProxmoxBackupJob[]): ProxmoxGuest[] {
|
||||
const enabledJobs = jobs.filter((j) => j.enabled);
|
||||
function splitIds(csv: string | null): string[] {
|
||||
return (csv ?? "").split(",").map((s) => s.trim()).filter(Boolean);
|
||||
}
|
||||
function isCovered(guest: ProxmoxGuest): boolean {
|
||||
return enabledJobs.some((job) => {
|
||||
if (job.node && job.node !== guest.node) return false;
|
||||
if (job.allGuests) return !splitIds(job.exclude).includes(String(guest.vmid));
|
||||
return splitIds(job.vmids).includes(String(guest.vmid));
|
||||
});
|
||||
}
|
||||
return guests.filter((g) => !isCovered(g));
|
||||
}
|
||||
|
||||
export interface ProxmoxAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
listGuests(): Promise<ProxmoxGuest[]>;
|
||||
|
||||
@@ -18,7 +18,7 @@ import { createTailscaleAdapter, isKeyExpiringSoon } from "../integrations/tails
|
||||
import { createGiteaAdapter } from "../integrations/gitea/adapter.js";
|
||||
import { createDockhandAdapter } from "../integrations/dockhand/adapter.js";
|
||||
import { createSemaphoreAdapter } from "../integrations/semaphore/adapter.js";
|
||||
import { createProxmoxAdapter } from "../integrations/proxmox/adapter.js";
|
||||
import { createProxmoxAdapter, guestsWithoutBackupCoverage } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter } from "../integrations/synology/adapter.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
@@ -669,8 +669,12 @@ integrationsRouter.get("/:id/proxmox/backups", asyncHandler(async (req, res) =>
|
||||
if (!found) return;
|
||||
|
||||
try {
|
||||
const [jobs, tasks] = await Promise.all([found.adapter.listBackupJobs(), found.adapter.listRecentBackupTasks()]);
|
||||
res.json({ jobs, tasks });
|
||||
const [jobs, tasks, guests] = await Promise.all([
|
||||
found.adapter.listBackupJobs(),
|
||||
found.adapter.listRecentBackupTasks(),
|
||||
found.adapter.listGuests(),
|
||||
]);
|
||||
res.json({ jobs, tasks, uncoveredGuests: guestsWithoutBackupCoverage(guests, jobs) });
|
||||
} catch (err) {
|
||||
res.status(502).json({ error: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
|
||||
@@ -257,6 +257,18 @@ export async function notifyProxmoxBackupFailure(
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyProxmoxUncoveredGuests(
|
||||
uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[],
|
||||
): Promise<void> {
|
||||
if (uncovered.length === 0) return;
|
||||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||||
const lines = uncovered.map((u) => `${u.guestName} (#${u.vmid}) on ${u.node} [${u.integrationName}]`);
|
||||
await notify(
|
||||
"Homelab Manager — Proxmox Guests Without Backup Coverage",
|
||||
`${uncovered.length} guest${uncovered.length !== 1 ? "s are" : " is"} not covered by any enabled backup job:\n\n${lines.join("\n")}`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyTailscaleKeyExpiry(
|
||||
expiring: { integrationName: string; deviceLabel: string; daysLeft: number }[],
|
||||
): Promise<void> {
|
||||
|
||||
@@ -3,8 +3,8 @@ import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxBackupTask } from "../integrations/proxmox/adapter.js";
|
||||
import { notifyProxmoxBackupFailure } from "./notify.js";
|
||||
import { createProxmoxAdapter, guestsWithoutBackupCoverage, type ProxmoxBackupTask } from "../integrations/proxmox/adapter.js";
|
||||
import { notifyProxmoxBackupFailure, notifyProxmoxUncoveredGuests } from "./notify.js";
|
||||
import { getSettings, getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
const LAST_RUN_FLAG = "proxmoxBackupCheckLastRunDate";
|
||||
@@ -16,13 +16,18 @@ async function checkProxmoxBackups(): Promise<void> {
|
||||
.where(and(eq(integrations.type, "proxmox"), eq(integrations.enabled, true)));
|
||||
|
||||
const failures: { integrationName: string; node: string; guestId: string | null; status: string }[] = [];
|
||||
const uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[] = [];
|
||||
|
||||
for (const row of rows) {
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(row.id);
|
||||
if (!loaded) continue;
|
||||
const adapter = createProxmoxAdapter(loaded.config as any);
|
||||
const tasks = await adapter.listRecentBackupTasks();
|
||||
const [tasks, jobs, guests] = await Promise.all([
|
||||
adapter.listRecentBackupTasks(),
|
||||
adapter.listBackupJobs(),
|
||||
adapter.listGuests(),
|
||||
]);
|
||||
|
||||
// Each node runs its own backup schedule independently, so check the
|
||||
// most recent run per node rather than only the single most recent
|
||||
@@ -39,12 +44,17 @@ async function checkProxmoxBackups(): Promise<void> {
|
||||
failures.push({ integrationName: row.name, node, guestId: task.guestId, status: task.status });
|
||||
}
|
||||
}
|
||||
|
||||
for (const g of guestsWithoutBackupCoverage(guests, jobs)) {
|
||||
uncovered.push({ integrationName: row.name, guestName: g.name, vmid: g.vmid, node: g.node });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[proxmoxBackup] check failed for integration ${row.id}:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
await notifyProxmoxBackupFailure(failures);
|
||||
await notifyProxmoxUncoveredGuests(uncovered);
|
||||
}
|
||||
|
||||
async function checkProxmoxBackupsOnce(): Promise<void> {
|
||||
|
||||
@@ -541,6 +541,7 @@ export interface ProxmoxBackupTask {
|
||||
export interface ProxmoxBackupsResponse {
|
||||
jobs: ProxmoxBackupJob[];
|
||||
tasks: ProxmoxBackupTask[];
|
||||
uncoveredGuests: ProxmoxGuest[];
|
||||
}
|
||||
|
||||
export interface SynologyVolume {
|
||||
|
||||
@@ -95,6 +95,7 @@ export default function Proxmox({ user }: { user: CurrentUser }) {
|
||||
|
||||
const [backupJobs, setBackupJobs] = useState<ProxmoxBackupJob[] | null>(null);
|
||||
const [backupTasks, setBackupTasks] = useState<ProxmoxBackupTask[] | null>(null);
|
||||
const [uncoveredGuests, setUncoveredGuests] = useState<ProxmoxGuest[] | null>(null);
|
||||
const [backupsError, setBackupsError] = useState<string | null>(null);
|
||||
const [loadingBackups, setLoadingBackups] = useState(false);
|
||||
|
||||
@@ -140,6 +141,7 @@ export default function Proxmox({ user }: { user: CurrentUser }) {
|
||||
.then((res) => {
|
||||
setBackupJobs(res.jobs);
|
||||
setBackupTasks(res.tasks);
|
||||
setUncoveredGuests(res.uncoveredGuests);
|
||||
})
|
||||
.catch((err) => setBackupsError(err instanceof Error ? err.message : String(err)))
|
||||
.finally(() => setLoadingBackups(false));
|
||||
@@ -155,6 +157,7 @@ export default function Proxmox({ user }: { user: CurrentUser }) {
|
||||
setNodeStats(null);
|
||||
setBackupJobs(null);
|
||||
setBackupTasks(null);
|
||||
setUncoveredGuests(null);
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [selectedId]);
|
||||
@@ -349,6 +352,12 @@ export default function Proxmox({ user }: { user: CurrentUser }) {
|
||||
{backupTasks[0].status}
|
||||
</div>
|
||||
)}
|
||||
{uncoveredGuests && uncoveredGuests.length > 0 && (
|
||||
<div className="alert alert-warning">
|
||||
{uncoveredGuests.length} guest{uncoveredGuests.length !== 1 ? "s are" : " is"} not covered by
|
||||
any enabled backup job: {uncoveredGuests.map((g) => `${g.name} (#${g.vmid})`).join(", ")}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="text-secondary small mb-1">Scheduled jobs</div>
|
||||
{backupJobs && backupJobs.length > 0 ? (
|
||||
@@ -382,6 +391,37 @@ export default function Proxmox({ user }: { user: CurrentUser }) {
|
||||
!backupsError && <div className="text-secondary small mb-4">No backup jobs configured in Proxmox.</div>
|
||||
)}
|
||||
|
||||
<div className="text-secondary small mb-1">Guests without backup coverage</div>
|
||||
{uncoveredGuests && uncoveredGuests.length > 0 ? (
|
||||
<table className="table table-sm table-vcenter mb-4">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Node</th>
|
||||
<th>Type</th>
|
||||
<th>Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{uncoveredGuests.map((g) => (
|
||||
<tr key={`${g.node}:${g.type}:${g.vmid}`}>
|
||||
<td>
|
||||
{g.name} <span className="text-secondary">#{g.vmid}</span>
|
||||
</td>
|
||||
<td className="text-secondary">{g.node}</td>
|
||||
<td>
|
||||
<span className="badge bg-blue-lt">{g.type === "qemu" ? "VM" : "LXC"}</span>
|
||||
</td>
|
||||
<td>{proxmoxStatusBadge(g.status)}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
) : (
|
||||
!backupsError &&
|
||||
uncoveredGuests && <div className="text-secondary small mb-4">Every VM/LXC is covered by at least one enabled backup job.</div>
|
||||
)}
|
||||
|
||||
<div className="text-secondary small mb-1">Recent runs</div>
|
||||
{backupTasks && backupTasks.length > 0 ? (
|
||||
<table className="table table-sm table-vcenter mb-0">
|
||||
|
||||
@@ -519,7 +519,7 @@ export default function NotificationSettings() {
|
||||
{ key: "secretCheck" as const, label: "Secret expiry reminder" },
|
||||
{ key: "tailscaleKeyCheck" as const, label: "Tailscale key expiry reminder" },
|
||||
{ key: "dockerUpdateCheck" as const, label: "Docker image update available" },
|
||||
{ key: "proxmoxBackupCheck" as const, label: "Proxmox backup failed" },
|
||||
{ key: "proxmoxBackupCheck" as const, label: "Proxmox backup failed or a guest has no coverage" },
|
||||
{ key: "integrationFailureAlerts" as const, label: "Integration/DNS provider failing repeatedly" },
|
||||
].map(({ key, label }) => (
|
||||
<label key={key} className="form-check mb-2">
|
||||
|
||||
Reference in New Issue
Block a user