Surface Proxmox backup job status, with a daily failure notification

Proxmox already runs vzdump backups, but nothing in the app said
whether they were actually succeeding — a silent backup failure is
one of the more dangerous blind spots a homelab admin can have. Adds
a "Backups" card to the Proxmox page: configured backup job
schedules (storage target, which guests, enabled/disabled) from
GET /cluster/backup, and recent vzdump task history per node from
GET /nodes/{node}/tasks?typefilter=vzdump, with a banner at the top
if the most recent run didn't succeed.

New "Proxmox backup failed" notification toggle under Settings ->
Notifications, on the same daily schedule as the other checks. The
scheduler checks each node's own most-recent vzdump run independently
(not just the single most recent task overall) so one node's healthy
backup can't mask another node's failing one in a multi-node cluster.

Known limitation, documented in the adapter's own header comment:
Proxmox's task list doesn't reliably expose which specific guest
failed within an "all guests" job — only the task's own log text has
that — so this surfaces job- and task-level status rather than
guessing at per-guest outcomes.

Verified against a fake Proxmox server (real self-signed HTTPS, since
the adapter's node:https usage can't be monkey-patched under ESM)
reproducing the documented /cluster/backup and task-list response
shapes: job parsing (all-guests+exclude vs specific-vmids+disabled)
correct, task OK/failure parsing correct, and the critical multi-node
scenario confirmed — one node's failing latest run flagged, the
other's healthy latest run correctly left alone, with exactly one
notification of the right content. This reproduces Proxmox's
documented API shape rather than a live-verified one; flag if the
real cluster's response differs in some way this didn't anticipate.
Confirmed the real dev database's mtime was untouched throughout.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

server/src/services/proxmoxBackupScheduler.ts

@@ -0,0 +1,85 @@
+import schedule from "node-schedule";
+import { and, eq } from "drizzle-orm";
+import { db } from "../db/client.js";
+import { integrations } from "../db/schema.js";
+import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
+import { createProxmoxAdapter, type ProxmoxBackupTask } from "../integrations/proxmox/adapter.js";
+import { notifyProxmoxBackupFailure } from "./notify.js";
+import { getSettings, getInternalFlag, setInternalFlag } from "./settingsStore.js";
+
+const LAST_RUN_FLAG = "proxmoxBackupCheckLastRunDate";
+
+async function checkProxmoxBackups(): Promise<void> {
+  const rows = await db
+    .select({ id: integrations.id, name: integrations.name })
+    .from(integrations)
+    .where(and(eq(integrations.type, "proxmox"), eq(integrations.enabled, true)));
+
+  const failures: { integrationName: string; node: string; guestId: string | null; status: string }[] = [];
+
+  for (const row of rows) {
+    try {
+      const loaded = await loadIntegrationConfig(row.id);
+      if (!loaded) continue;
+      const adapter = createProxmoxAdapter(loaded.config as any);
+      const tasks = await adapter.listRecentBackupTasks();
+
+      // Each node runs its own backup schedule independently, so check the
+      // most recent run per node rather than only the single most recent
+      // task overall — otherwise a failing node could be masked by a
+      // healthier node's more recent run.
+      const latestByNode = new Map<string, ProxmoxBackupTask>();
+      for (const t of tasks) {
+        const existing = latestByNode.get(t.node);
+        if (!existing || t.startTime > existing.startTime) latestByNode.set(t.node, t);
+      }
+
+      for (const [node, task] of latestByNode) {
+        if (!task.ok && task.status !== "running") {
+          failures.push({ integrationName: row.name, node, guestId: task.guestId, status: task.status });
+        }
+      }
+    } catch (err) {
+      console.error(`[proxmoxBackup] check failed for integration ${row.id}:`, err);
+    }
+  }
+
+  await notifyProxmoxBackupFailure(failures);
+}
+
+async function checkProxmoxBackupsOnce(): Promise<void> {
+  const today = new Date().toDateString();
+  const lastRun = await getInternalFlag(LAST_RUN_FLAG);
+  if (lastRun === today) return;
+  await setInternalFlag(LAST_RUN_FLAG, today);
+  await checkProxmoxBackups();
+}
+
+function cronFromTime(time: string): string {
+  const [h, m] = time.split(":").map(Number);
+  return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} * * *`;
+}
+
+let currentJob: schedule.Job | null = null;
+
+/** (Re)schedules the daily Proxmox backup-failure check per the current notification settings. Call again after settings change. */
+export async function scheduleProxmoxBackupCheck(): Promise<void> {
+  if (currentJob) {
+    currentJob.cancel();
+    currentJob = null;
+  }
+  const { notifications } = await getSettings();
+  currentJob = schedule.scheduleJob({ rule: cronFromTime(notifications.secretCheckTime), tz: notifications.timezone }, () => {
+    setInternalFlag(LAST_RUN_FLAG, "").catch(() => {});
+    getSettings().then(({ notifications: n }) => {
+      if (n.proxmoxBackupCheck) checkProxmoxBackups().catch((err) => console.error("[proxmoxBackup] check failed:", err));
+    });
+  });
+  console.log(`Proxmox backup check scheduled at ${notifications.secretCheckTime} (${notifications.timezone})`);
+}
+
+/** Runs once at startup (skipped if already run today), then arms the daily schedule. */
+export async function initProxmoxBackupScheduler(): Promise<void> {
+  await checkProxmoxBackupsOnce();
+  await scheduleProxmoxBackupCheck();
+}