Add a Proxmox Backup Server integration: datastore/snapshot verification status

Proxmox VE already shows whether the last vzdump push to PBS succeeded, but
has no visibility into PBS's own backup verification, GC/prune health, or
host status. This adds PBS as its own integration (own adapter, page, nav
entry, and Dashboard widget) that reads datastore usage and, for every
stored snapshot, its verification state directly from PBS.

A new daily check (mirroring the existing Proxmox backup-failure check)
notifies when a snapshot has failed verification or a datastore couldn't be
read, with its own toggle in Settings -> Notifications and its own
maintenance-window silencing.

Not verified against a live PBS instance — built from PBS's published API
docs and a scratch test against a mocked PBS server exercising the adapter's
parsing and auth-header format (PBSAPIToken uses a colon separator, unlike
PVE's PVEAPIToken which uses =). See INTEGRATIONS.md for details and the
"not verified" caveat.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

server/src/services/notify.ts

@@ -277,6 +277,22 @@ export async function notifyProxmoxUncoveredGuests(
   );
 }
 
+export async function notifyPbsVerificationFailed(
+  failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[],
+): Promise<void> {
+  if (failures.length === 0) return;
+  if (!(await eventEnabled("pbsVerificationCheck"))) return;
+  const lines = failures.map((f) =>
+    f.error
+      ? `${f.datastore} [${f.integrationName}]: couldn't be read — ${f.error}`
+      : `${f.datastore} [${f.integrationName}]: ${f.failedCount} snapshot${f.failedCount !== 1 ? "s" : ""} failed verification`,
+  );
+  await notify(
+    "Homelab Manager — Proxmox Backup Server Verification Failed",
+    `${failures.length} datastore${failures.length !== 1 ? "s have" : " has"} a problem:\n\n${lines.join("\n")}`,
+  );
+}
+
 export async function notifyHealthIssues(issues: { message: string }[]): Promise<void> {
   if (issues.length === 0) return;
   if (!(await eventEnabled("healthAlerts"))) return;

server/src/services/pbsVerificationScheduler.ts

@@ -0,0 +1,80 @@
+import schedule from "node-schedule";
+import { and, eq } from "drizzle-orm";
+import { db } from "../db/client.js";
+import { integrations } from "../db/schema.js";
+import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
+import { createPbsAdapter } from "../integrations/pbs/adapter.js";
+import { notifyPbsVerificationFailed } from "./notify.js";
+import { getSettings, getInternalFlag, setInternalFlag } from "./settingsStore.js";
+import { isInMaintenance } from "./maintenance.js";
+
+const LAST_RUN_FLAG = "pbsVerificationCheckLastRunDate";
+
+async function checkPbsVerification(): Promise<void> {
+  const rows = await db
+    .select({ id: integrations.id, name: integrations.name })
+    .from(integrations)
+    .where(and(eq(integrations.type, "pbs"), eq(integrations.enabled, true)));
+
+  const failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[] = [];
+
+  for (const row of rows) {
+    // A PBS host being worked on can't be reached reliably; this check is daily, so tomorrow's pass covers it.
+    if (await isInMaintenance("integration", row.id)) continue;
+    try {
+      const loaded = await loadIntegrationConfig(row.id);
+      if (!loaded) continue;
+      const adapter = createPbsAdapter(loaded.config as any);
+      const datastores = await adapter.listDatastores();
+
+      for (const d of datastores) {
+        if (d.error) {
+          failures.push({ integrationName: row.name, datastore: d.name, failedCount: 0, error: d.error });
+        } else if (d.failedCount > 0) {
+          failures.push({ integrationName: row.name, datastore: d.name, failedCount: d.failedCount, error: null });
+        }
+      }
+    } catch (err) {
+      console.error(`[pbsVerification] check failed for integration ${row.id}:`, err);
+    }
+  }
+
+  await notifyPbsVerificationFailed(failures);
+}
+
+async function checkPbsVerificationOnce(): Promise<void> {
+  const today = new Date().toDateString();
+  const lastRun = await getInternalFlag(LAST_RUN_FLAG);
+  if (lastRun === today) return;
+  await setInternalFlag(LAST_RUN_FLAG, today);
+  await checkPbsVerification();
+}
+
+function cronFromTime(time: string): string {
+  const [h, m] = time.split(":").map(Number);
+  return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} * * *`;
+}
+
+let currentJob: schedule.Job | null = null;
+
+/** (Re)schedules the daily PBS verification-failure check per the current notification settings. Call again after settings change. */
+export async function schedulePbsVerificationCheck(): Promise<void> {
+  if (currentJob) {
+    currentJob.cancel();
+    currentJob = null;
+  }
+  const { notifications } = await getSettings();
+  currentJob = schedule.scheduleJob({ rule: cronFromTime(notifications.secretCheckTime), tz: notifications.timezone }, () => {
+    setInternalFlag(LAST_RUN_FLAG, "").catch(() => {});
+    getSettings().then(({ notifications: n }) => {
+      if (n.pbsVerificationCheck) checkPbsVerification().catch((err) => console.error("[pbsVerification] check failed:", err));
+    });
+  });
+  console.log(`PBS verification check scheduled at ${notifications.secretCheckTime} (${notifications.timezone})`);
+}
+
+/** Runs once at startup (skipped if already run today), then arms the daily schedule. */
+export async function initPbsVerificationScheduler(): Promise<void> {
+  await checkPbsVerificationOnce();
+  await schedulePbsVerificationCheck();
+}

server/src/services/settingsStore.ts

@@ -42,6 +42,8 @@ export interface NotificationEvents {
   tailscaleKeyCheck: boolean;
   dockerUpdateCheck: boolean;
   proxmoxBackupCheck: boolean;
+  /** Daily reminder while a Proxmox Backup Server snapshot has failed verification, or a datastore couldn't be read at all. */
+  pbsVerificationCheck: boolean;
   healthAlerts: boolean;
   /** A Semaphore template or Gitea repo whose latest run failed, and when it succeeds again. */
   automationAlerts: boolean;
@@ -125,6 +127,7 @@ const DEFAULTS: AppSettings = {
     tailscaleKeyCheck: true,
     dockerUpdateCheck: true,
     proxmoxBackupCheck: true,
+    pbsVerificationCheck: true,
     healthAlerts: true,
     automationAlerts: true,
     domainExpiryCheck: true,