Surface Proxmox backup job status, with a daily failure notification

Proxmox already runs vzdump backups, but nothing in the app said
whether they were actually succeeding — a silent backup failure is
one of the more dangerous blind spots a homelab admin can have. Adds
a "Backups" card to the Proxmox page: configured backup job
schedules (storage target, which guests, enabled/disabled) from
GET /cluster/backup, and recent vzdump task history per node from
GET /nodes/{node}/tasks?typefilter=vzdump, with a banner at the top
if the most recent run didn't succeed.

New "Proxmox backup failed" notification toggle under Settings ->
Notifications, on the same daily schedule as the other checks. The
scheduler checks each node's own most-recent vzdump run independently
(not just the single most recent task overall) so one node's healthy
backup can't mask another node's failing one in a multi-node cluster.

Known limitation, documented in the adapter's own header comment:
Proxmox's task list doesn't reliably expose which specific guest
failed within an "all guests" job — only the task's own log text has
that — so this surfaces job- and task-level status rather than
guessing at per-guest outcomes.

Verified against a fake Proxmox server (real self-signed HTTPS, since
the adapter's node:https usage can't be monkey-patched under ESM)
reproducing the documented /cluster/backup and task-list response
shapes: job parsing (all-guests+exclude vs specific-vmids+disabled)
correct, task OK/failure parsing correct, and the critical multi-node
scenario confirmed — one node's failing latest run flagged, the
other's healthy latest run correctly left alone, with exactly one
notification of the right content. This reproduces Proxmox's
documented API shape rather than a live-verified one; flag if the
real cluster's response differs in some way this didn't anticipate.
Confirmed the real dev database's mtime was untouched throughout.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

server/src/index.ts

@@ -27,6 +27,7 @@ import { initSecretExpiryScheduler } from "./services/secretExpiryScheduler.js";
 import { initTailscaleKeyExpiryScheduler } from "./services/tailscaleKeyExpiryScheduler.js";
 import { initLogRetentionScheduler } from "./services/logRetentionScheduler.js";
 import { initDockerUpdateScheduler } from "./services/dockerUpdateScheduler.js";
+import { initProxmoxBackupScheduler } from "./services/proxmoxBackupScheduler.js";
 
 warnIfAuthNotConfigured();
 await runMigrations();
@@ -34,6 +35,7 @@ await initSecretExpiryScheduler();
 await initTailscaleKeyExpiryScheduler();
 await initLogRetentionScheduler();
 await initDockerUpdateScheduler();
+await initProxmoxBackupScheduler();
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const webDist = join(__dirname, "..", "..", "web", "dist");

server/src/integrations/proxmox/adapter.ts

@@ -9,7 +9,13 @@
  * (https://pve.proxmox.com/pve-docs/api-viewer/apidoc.js): GET /nodes, GET
  * /nodes/{node}/qemu, GET /nodes/{node}/lxc, and POST
  * /nodes/{node}/{qemu,lxc}/{vmid}/status/{start,stop,reboot} (all
- * token-auth-eligible per that spec's "allowtoken" flag).
+ * token-auth-eligible per that spec's "allowtoken" flag). Backup visibility
+ * uses GET /cluster/backup (job schedules — requires Sys.Audit on /) and GET
+ * /nodes/{node}/tasks?typefilter=vzdump (run history — same Sys.Audit as the
+ * existing node-stats card already needs). Per-guest outcome within an
+ * "all guests" job isn't reliably exposed by the task list itself (only the
+ * task's own log text has that), so this surfaces job-level and task-level
+ * status rather than guessing at per-guest results.
  *
  * Proxmox commonly runs with a self-signed certificate in homelab setups, so
  * (like the cPanel DNS adapter) this uses node:https directly rather than
@@ -92,6 +98,37 @@ export interface ProxmoxNodeStats {
   storages: ProxmoxStorage[];
 }
 
+export interface ProxmoxBackupJob {
+  id: string;
+  enabled: boolean;
+  schedule: string;
+  storage: string;
+  /** null = a cluster-wide job not pinned to one node. */
+  node: string | null;
+  allGuests: boolean;
+  /** Comma-separated guest IDs this job backs up — present when allGuests is false. */
+  vmids: string | null;
+  /** Comma-separated guest IDs excluded from an allGuests job. */
+  exclude: string | null;
+}
+
+export interface ProxmoxBackupTask {
+  node: string;
+  upid: string;
+  /**
+   * Proxmox's own task "id" field — for a single-guest vzdump run this is
+   * that guest's vmid, but for an "all guests" job it can be blank (the
+   * per-guest outcomes only exist in the task's own log text, which this
+   * doesn't fetch/parse) — shown as-is rather than guessed at.
+   */
+  guestId: string | null;
+  /** "OK", an error string, or "running" for a task with no endtime yet. */
+  status: string;
+  ok: boolean;
+  startTime: string; // ISO
+  endTime: string | null; // ISO, null while still running
+}
+
 export interface ProxmoxAdapter {
   ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
   listGuests(): Promise<ProxmoxGuest[]>;
@@ -102,6 +139,9 @@ export interface ProxmoxAdapter {
   restartGuest(node: string, type: ProxmoxGuestType, vmid: number): Promise<void>;
   /** Graceful shutdown (ACPI power event for a VM, SIGTERM-then-wait for a container) — unlike stopGuest, this asks the guest OS to shut itself down. */
   shutdownGuest(node: string, type: ProxmoxGuestType, vmid: number): Promise<void>;
+  listBackupJobs(): Promise<ProxmoxBackupJob[]>;
+  /** Most recent vzdump task runs across every online node, newest first. */
+  listRecentBackupTasks(limitPerNode?: number): Promise<ProxmoxBackupTask[]>;
 }
 
 function parseSizeToBytes(size: string): number | null {
@@ -417,6 +457,44 @@ export function createProxmoxAdapter(config: ProxmoxConfig): ProxmoxAdapter {
     );
   }
 
+  async function listBackupJobs(): Promise<ProxmoxBackupJob[]> {
+    const data = await api("GET", "/cluster/backup");
+    return (Array.isArray(data) ? data : []).map((j: any) => ({
+      id: j.id,
+      enabled: j.enabled !== 0 && j.enabled !== "0",
+      schedule: j.schedule ?? "",
+      storage: j.storage ?? "",
+      node: j.node ?? null,
+      allGuests: j.all === 1 || j.all === "1",
+      vmids: typeof j.vmid === "string" ? j.vmid : null,
+      exclude: typeof j.exclude === "string" ? j.exclude : null,
+    }));
+  }
+
+  async function listRecentBackupTasks(limitPerNode = 20): Promise<ProxmoxBackupTask[]> {
+    const nodes = await listNodes();
+    const perNode = await Promise.all(
+      nodes.map(async (node) => {
+        try {
+          const data = await api("GET", `/nodes/${node}/tasks?typefilter=vzdump&limit=${limitPerNode}`);
+          return (Array.isArray(data) ? data : []).map((t: any) => ({
+            node,
+            upid: t.upid,
+            guestId: t.id || null,
+            status: t.status ?? (t.endtime ? "unknown" : "running"),
+            ok: t.status === "OK",
+            startTime: new Date(t.starttime * 1000).toISOString(),
+            endTime: typeof t.endtime === "number" ? new Date(t.endtime * 1000).toISOString() : null,
+          }));
+        } catch {
+          // one unreachable/offline node shouldn't take down the whole backup view
+          return [];
+        }
+      }),
+    );
+    return perNode.flat().sort((a, b) => b.startTime.localeCompare(a.startTime));
+  }
+
   async function statusAction(node: string, type: ProxmoxGuestType, vmid: number, action: string): Promise<void> {
     await api("POST", `/nodes/${node}/${type}/${vmid}/status/${action}`);
   }
@@ -445,5 +523,7 @@ export function createProxmoxAdapter(config: ProxmoxConfig): ProxmoxAdapter {
     stopGuest,
     restartGuest,
     shutdownGuest,
+    listBackupJobs,
+    listRecentBackupTasks,
   });
 }

server/src/routes/integrations.ts

@@ -664,6 +664,18 @@ integrationsRouter.get("/:id/proxmox/nodes", asyncHandler(async (req, res) => {
   }
 }));
 
+integrationsRouter.get("/:id/proxmox/backups", asyncHandler(async (req, res) => {
+  const found = await requireProxmoxAdapter(req, res);
+  if (!found) return;
+
+  try {
+    const [jobs, tasks] = await Promise.all([found.adapter.listBackupJobs(), found.adapter.listRecentBackupTasks()]);
+    res.json({ jobs, tasks });
+  } catch (err) {
+    res.status(502).json({ error: err instanceof Error ? err.message : String(err) });
+  }
+}));
+
 const proxmoxActions = ["start", "stop", "restart", "shutdown"] as const;
 
 for (const action of proxmoxActions) {

server/src/routes/settings.ts

@@ -6,6 +6,7 @@ import { getSettings, updateSettings } from "../services/settingsStore.js";
 import { scheduleSecretExpiryCheck } from "../services/secretExpiryScheduler.js";
 import { scheduleTailscaleKeyExpiryCheck } from "../services/tailscaleKeyExpiryScheduler.js";
 import { scheduleDockerUpdateCheck } from "../services/dockerUpdateScheduler.js";
+import { scheduleProxmoxBackupCheck } from "../services/proxmoxBackupScheduler.js";
 import { scheduleLogRetentionPurge } from "../services/logRetentionScheduler.js";
 import { purgeOldLogs } from "../services/logRetention.js";
 import { testGotify, testNtfy, testSmtp, testWebhook } from "../services/notify.js";
@@ -64,6 +65,7 @@ const updateSchema = z.object({
       secretCheck: z.boolean(),
       tailscaleKeyCheck: z.boolean(),
       dockerUpdateCheck: z.boolean(),
+      proxmoxBackupCheck: z.boolean(),
       secretCheckTime: z.string().regex(/^\d{2}:\d{2}$/),
       timezone: z.string(),
       integrationFailureAlerts: z.boolean(),
@@ -95,6 +97,7 @@ settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
     await scheduleSecretExpiryCheck();
     await scheduleTailscaleKeyExpiryCheck();
     await scheduleDockerUpdateCheck();
+    await scheduleProxmoxBackupCheck();
   }
   if (parsed.data.logRetention) {
     await scheduleLogRetentionPurge();

server/src/services/notify.ts

@@ -192,6 +192,20 @@ export async function notifyDockerUpdates(
   );
 }
 
+export async function notifyProxmoxBackupFailure(
+  failures: { integrationName: string; node: string; guestId: string | null; status: string }[],
+): Promise<void> {
+  if (failures.length === 0) return;
+  if (!(await eventEnabled("proxmoxBackupCheck"))) return;
+  const lines = failures.map(
+    (f) => `${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} [${f.integrationName}]: ${f.status}`,
+  );
+  await notify(
+    "Homelab Manager — Proxmox Backup Failed",
+    `${failures.length} node${failures.length !== 1 ? "s have" : " has"} a failing most-recent backup run:\n\n${lines.join("\n")}`,
+  );
+}
+
 export async function notifyTailscaleKeyExpiry(
   expiring: { integrationName: string; deviceLabel: string; daysLeft: number }[],
 ): Promise<void> {

server/src/services/proxmoxBackupScheduler.ts

@@ -0,0 +1,85 @@
+import schedule from "node-schedule";
+import { and, eq } from "drizzle-orm";
+import { db } from "../db/client.js";
+import { integrations } from "../db/schema.js";
+import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
+import { createProxmoxAdapter, type ProxmoxBackupTask } from "../integrations/proxmox/adapter.js";
+import { notifyProxmoxBackupFailure } from "./notify.js";
+import { getSettings, getInternalFlag, setInternalFlag } from "./settingsStore.js";
+
+const LAST_RUN_FLAG = "proxmoxBackupCheckLastRunDate";
+
+async function checkProxmoxBackups(): Promise<void> {
+  const rows = await db
+    .select({ id: integrations.id, name: integrations.name })
+    .from(integrations)
+    .where(and(eq(integrations.type, "proxmox"), eq(integrations.enabled, true)));
+
+  const failures: { integrationName: string; node: string; guestId: string | null; status: string }[] = [];
+
+  for (const row of rows) {
+    try {
+      const loaded = await loadIntegrationConfig(row.id);
+      if (!loaded) continue;
+      const adapter = createProxmoxAdapter(loaded.config as any);
+      const tasks = await adapter.listRecentBackupTasks();
+
+      // Each node runs its own backup schedule independently, so check the
+      // most recent run per node rather than only the single most recent
+      // task overall — otherwise a failing node could be masked by a
+      // healthier node's more recent run.
+      const latestByNode = new Map<string, ProxmoxBackupTask>();
+      for (const t of tasks) {
+        const existing = latestByNode.get(t.node);
+        if (!existing || t.startTime > existing.startTime) latestByNode.set(t.node, t);
+      }
+
+      for (const [node, task] of latestByNode) {
+        if (!task.ok && task.status !== "running") {
+          failures.push({ integrationName: row.name, node, guestId: task.guestId, status: task.status });
+        }
+      }
+    } catch (err) {
+      console.error(`[proxmoxBackup] check failed for integration ${row.id}:`, err);
+    }
+  }
+
+  await notifyProxmoxBackupFailure(failures);
+}
+
+async function checkProxmoxBackupsOnce(): Promise<void> {
+  const today = new Date().toDateString();
+  const lastRun = await getInternalFlag(LAST_RUN_FLAG);
+  if (lastRun === today) return;
+  await setInternalFlag(LAST_RUN_FLAG, today);
+  await checkProxmoxBackups();
+}
+
+function cronFromTime(time: string): string {
+  const [h, m] = time.split(":").map(Number);
+  return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} * * *`;
+}
+
+let currentJob: schedule.Job | null = null;
+
+/** (Re)schedules the daily Proxmox backup-failure check per the current notification settings. Call again after settings change. */
+export async function scheduleProxmoxBackupCheck(): Promise<void> {
+  if (currentJob) {
+    currentJob.cancel();
+    currentJob = null;
+  }
+  const { notifications } = await getSettings();
+  currentJob = schedule.scheduleJob({ rule: cronFromTime(notifications.secretCheckTime), tz: notifications.timezone }, () => {
+    setInternalFlag(LAST_RUN_FLAG, "").catch(() => {});
+    getSettings().then(({ notifications: n }) => {
+      if (n.proxmoxBackupCheck) checkProxmoxBackups().catch((err) => console.error("[proxmoxBackup] check failed:", err));
+    });
+  });
+  console.log(`Proxmox backup check scheduled at ${notifications.secretCheckTime} (${notifications.timezone})`);
+}
+
+/** Runs once at startup (skipped if already run today), then arms the daily schedule. */
+export async function initProxmoxBackupScheduler(): Promise<void> {
+  await checkProxmoxBackupsOnce();
+  await scheduleProxmoxBackupCheck();
+}

server/src/services/settingsStore.ts

@@ -41,7 +41,8 @@ export interface NotificationEvents {
   secretCheck: boolean;
   tailscaleKeyCheck: boolean;
   dockerUpdateCheck: boolean;
-  secretCheckTime: string; // "HH:MM" — shared by the secret-expiry, Tailscale key-expiry, and Docker update checks
+  proxmoxBackupCheck: boolean;
+  secretCheckTime: string; // "HH:MM" — shared by the secret-expiry, Tailscale key-expiry, Docker update, and Proxmox backup checks
   timezone: string;
   integrationFailureAlerts: boolean;
   /** Consecutive failed calls (from the Diagnostic Log) before an integration/DNS provider is considered down. */
@@ -93,6 +94,7 @@ const DEFAULTS: AppSettings = {
     secretCheck: true,
     tailscaleKeyCheck: true,
     dockerUpdateCheck: true,
+    proxmoxBackupCheck: true,
     secretCheckTime: "08:00",
     timezone: "UTC",
     integrationFailureAlerts: true,