Add a Proxmox Backup Server integration: datastore/snapshot verification status

Proxmox VE already shows whether the last vzdump push to PBS succeeded, but
has no visibility into PBS's own backup verification, GC/prune health, or
host status. This adds PBS as its own integration (own adapter, page, nav
entry, and Dashboard widget) that reads datastore usage and, for every
stored snapshot, its verification state directly from PBS.

A new daily check (mirroring the existing Proxmox backup-failure check)
notifies when a snapshot has failed verification or a datastore couldn't be
read, with its own toggle in Settings -> Notifications and its own
maintenance-window silencing.

Not verified against a live PBS instance — built from PBS's published API
docs and a scratch test against a mocked PBS server exercising the adapter's
parsing and auth-header format (PBSAPIToken uses a colon separator, unlike
PVE's PVEAPIToken which uses =). See INTEGRATIONS.md for details and the
"not verified" caveat.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

server/src/db/schema.ts

@@ -288,6 +288,7 @@ export const integrationTypes = [
   "dockhand",
   "uptimekuma",
   "phpipam",
+  "pbs",
 ] as const;
 export type IntegrationType = (typeof integrationTypes)[number];
 

server/src/index.ts

@@ -33,6 +33,7 @@ import { initTailscaleKeyExpiryScheduler } from "./services/tailscaleKeyExpirySc
 import { initLogRetentionScheduler } from "./services/logRetentionScheduler.js";
 import { initDockerUpdateScheduler } from "./services/dockerUpdateScheduler.js";
 import { initProxmoxBackupScheduler } from "./services/proxmoxBackupScheduler.js";
+import { initPbsVerificationScheduler } from "./services/pbsVerificationScheduler.js";
 import { initQuietHoursScheduler } from "./services/quietHoursScheduler.js";
 import { initHealthScheduler } from "./services/healthScheduler.js";
 
@@ -43,6 +44,7 @@ await initTailscaleKeyExpiryScheduler();
 await initLogRetentionScheduler();
 await initDockerUpdateScheduler();
 await initProxmoxBackupScheduler();
+await initPbsVerificationScheduler();
 await initQuietHoursScheduler();
 await initHealthScheduler();
 

server/src/integrations/fieldSchemas.ts

@@ -56,6 +56,12 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
     { key: "token", label: "App token (API code)", secret: true, type: "password" },
     { key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
   ],
+  pbs: [
+    { key: "url", label: "Proxmox Backup Server URL", secret: false, placeholder: "https://pbs.example.lan:8007" },
+    { key: "tokenId", label: "API token ID", secret: false, placeholder: "root@pam!homelab-manager" },
+    { key: "tokenSecret", label: "API token secret", secret: true, type: "password" },
+    { key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
+  ],
 };
 
 /** Fixed base URL per integration type, stored on the row for display/reference. */

server/src/integrations/pbs/adapter.ts

@@ -0,0 +1,239 @@
+/**
+ * Proxmox Backup Server adapter — uses PBS's REST API (api2/json), the same overall shape as
+ * Proxmox VE's (both are built on the same Rust API framework), but a distinct product with its
+ * own auth scheme and endpoints. Requires config: url, tokenId, tokenSecret; optional: insecure
+ *
+ * Auth: `Authorization: PBSAPIToken=<tokenId>:<tokenSecret>` — note the colon, not the `=` PVE
+ * uses between the id and the secret; the id itself is the same shape either product uses
+ * ("user@realm!tokenname"). See https://pbs.proxmox.com/docs/user-management.html.
+ *
+ * PBS's dashboard/API is HTTPS-only (default port 8007) and, like Proxmox VE, commonly runs with
+ * a self-signed certificate in a homelab — hence the same "insecure" opt-out via node:https.
+ *
+ * There is no single "is this backup okay" flag anywhere in Proxmox VE — vzdump only reports that
+ * the push to the datastore finished, never whether the stored data still verifies. This adapter
+ * reads that directly from PBS: GET /admin/datastore lists the configured datastores, GET
+ * /admin/datastore/{store}/status gives its usage, and GET /admin/datastore/{store}/snapshots
+ * lists every stored backup with its own verification state — read from the snapshot data
+ * itself rather than by trying to correlate verify-job schedules with task-log entries, since the
+ * snapshot's own state is the ground truth and doesn't depend on guessing a task "worker type"
+ * string. GET /nodes/localhost/status gives the server's own CPU/RAM/disk — PBS is a single
+ * node, and "localhost" is the documented way to address it without needing its real hostname.
+ *
+ * Endpoints, the token header format, and the datastore/snapshot field names are cross-checked
+ * against PBS's own published documentation and API-derived community write-ups, but this has
+ * not been run against a live instance. Every field is read defensively (optional, independently
+ * type-checked), so a field PBS renames or omits in some version leaves that value blank rather
+ * than breaking the whole read.
+ */
+import * as https from "node:https";
+import { withDiagLogging } from "../../services/diagLog.js";
+
+export interface PbsConfig {
+  url: string;
+  tokenId: string;
+  tokenSecret: string;
+  insecure?: boolean;
+}
+
+export interface PbsFailedSnapshot {
+  backupType: string; // "vm" | "ct" | "host"
+  backupId: string;
+  /** Unix seconds. */
+  backupTime: number;
+}
+
+export interface PbsDatastore {
+  name: string;
+  comment: string | null;
+  totalBytes: number | null;
+  usedBytes: number | null;
+  availBytes: number | null;
+  /** null when the datastore couldn't be read at all (e.g. this token lacks Datastore.Audit on it) — distinct from "0 snapshots". */
+  error: string | null;
+  snapshotCount: number;
+  /** Verified and found bad. */
+  failedCount: number;
+  /** Present in the datastore but never checked by a verify job. */
+  unverifiedCount: number;
+  /** Newest snapshot across the whole datastore, if any (unix seconds). */
+  latestSnapshotAt: number | null;
+  /** Up to 20 of the most recent verification failures, newest first. */
+  recentFailures: PbsFailedSnapshot[];
+}
+
+export interface PbsNodeStatus {
+  cpuUsagePercent: number | null;
+  cpuCores: number | null;
+  memTotalBytes: number | null;
+  memUsedBytes: number | null;
+  rootfsTotalBytes: number | null;
+  rootfsUsedBytes: number | null;
+  uptime: number | null;
+}
+
+export interface PbsAdapter {
+  ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
+  listDatastores(): Promise<PbsDatastore[]>;
+  getNodeStatus(): Promise<PbsNodeStatus>;
+}
+
+interface RawResponse {
+  status: number;
+  text: () => string;
+}
+
+function request(url: string, insecure: boolean, headers: Record<string, string>): Promise<RawResponse> {
+  return new Promise((resolve, reject) => {
+    const parsed = new URL(url);
+    const req = https.request(
+      {
+        hostname: parsed.hostname,
+        port: parsed.port || 8007,
+        path: parsed.pathname + parsed.search,
+        method: "GET",
+        headers,
+        rejectUnauthorized: !insecure,
+      },
+      (res) => {
+        let body = "";
+        res.setEncoding("utf8");
+        res.on("data", (chunk) => {
+          body += chunk;
+        });
+        res.on("end", () => resolve({ status: res.statusCode ?? 0, text: () => body }));
+      },
+    );
+    req.on("error", reject);
+    req.end();
+  });
+}
+
+const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
+const str = (v: unknown): string | null => (typeof v === "string" && v !== "" ? v : null);
+
+export function createPbsAdapter(config: PbsConfig): PbsAdapter {
+  const insecure = config.insecure === true;
+
+  function base() {
+    return config.url.replace(/\/$/, "");
+  }
+
+  function headers() {
+    return { Authorization: `PBSAPIToken=${config.tokenId}:${config.tokenSecret}`, Accept: "application/json" };
+  }
+
+  async function api(path: string): Promise<any> {
+    const res = await request(`${base()}/api2/json${path}`, insecure, headers());
+    let data: any = null;
+    try {
+      data = res.text() ? JSON.parse(res.text()) : null;
+    } catch {
+      // non-JSON error page
+    }
+    if (res.status < 200 || res.status >= 300) {
+      const message = data?.errors ? JSON.stringify(data.errors) : data?.message;
+      throw new Error(message || `Proxmox Backup Server API error: HTTP ${res.status}`);
+    }
+    return data?.data;
+  }
+
+  async function listDatastoreNames(): Promise<{ name: string; comment: string | null }[]> {
+    const data = await api("/admin/datastore");
+    return (Array.isArray(data) ? data : [])
+      .map((d: any) => ({ name: str(d?.name ?? d?.store), comment: str(d?.comment) }))
+      .filter((d: { name: string | null }): d is { name: string; comment: string | null } => d.name !== null);
+  }
+
+  async function getDatastoreStatus(name: string): Promise<{ total: number | null; used: number | null; avail: number | null }> {
+    const data = await api(`/admin/datastore/${encodeURIComponent(name)}/status`);
+    return { total: num(data?.total), used: num(data?.used), avail: num(data?.avail) };
+  }
+
+  async function getSnapshotSummary(name: string) {
+    const data = await api(`/admin/datastore/${encodeURIComponent(name)}/snapshots`);
+    const snapshots = Array.isArray(data) ? data : [];
+
+    let failedCount = 0;
+    let unverifiedCount = 0;
+    let latestSnapshotAt: number | null = null;
+    const failures: PbsFailedSnapshot[] = [];
+
+    for (const s of snapshots) {
+      const backupTime = num(s?.["backup-time"]);
+      if (backupTime !== null && (latestSnapshotAt === null || backupTime > latestSnapshotAt)) latestSnapshotAt = backupTime;
+
+      const state = str(s?.verification?.state)?.toLowerCase() ?? null;
+      if (state === "failed") {
+        failedCount++;
+        const backupType = str(s?.["backup-type"]);
+        const backupId = str(s?.["backup-id"]);
+        if (backupType && backupId && backupTime !== null) failures.push({ backupType, backupId, backupTime });
+      } else if (state === null) {
+        unverifiedCount++;
+      }
+    }
+
+    failures.sort((a, b) => b.backupTime - a.backupTime);
+    return { snapshotCount: snapshots.length, failedCount, unverifiedCount, latestSnapshotAt, recentFailures: failures.slice(0, 20) };
+  }
+
+  async function listDatastores(): Promise<PbsDatastore[]> {
+    const names = await listDatastoreNames();
+    return Promise.all(
+      names.map(async ({ name, comment }): Promise<PbsDatastore> => {
+        try {
+          const [status, snapshots] = await Promise.all([getDatastoreStatus(name), getSnapshotSummary(name)]);
+          return {
+            name,
+            comment,
+            totalBytes: status.total,
+            usedBytes: status.used,
+            availBytes: status.avail,
+            error: null,
+            ...snapshots,
+          };
+        } catch (err) {
+          return {
+            name,
+            comment,
+            totalBytes: null,
+            usedBytes: null,
+            availBytes: null,
+            error: err instanceof Error ? err.message : String(err),
+            snapshotCount: 0,
+            failedCount: 0,
+            unverifiedCount: 0,
+            latestSnapshotAt: null,
+            recentFailures: [],
+          };
+        }
+      }),
+    );
+  }
+
+  async function getNodeStatus(): Promise<PbsNodeStatus> {
+    const data = await api("/nodes/localhost/status");
+    return {
+      cpuUsagePercent: num(data?.cpu) !== null ? num(data.cpu)! * 100 : null,
+      cpuCores: num(data?.cpuinfo?.cpus),
+      memTotalBytes: num(data?.memory?.total),
+      memUsedBytes: num(data?.memory?.used),
+      rootfsTotalBytes: num(data?.root?.total),
+      rootfsUsedBytes: num(data?.root?.used),
+      uptime: num(data?.uptime),
+    };
+  }
+
+  async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
+    const start = Date.now();
+    try {
+      await api("/admin/datastore");
+      return { ok: true, latencyMs: Date.now() - start };
+    } catch (err) {
+      return { ok: false, error: err instanceof Error ? err.message : String(err) };
+    }
+  }
+
+  return withDiagLogging("pbs", { ping, listDatastores, getNodeStatus });
+}

server/src/integrations/registry.ts

@@ -8,6 +8,7 @@ import { createProxmoxAdapter } from "./proxmox/adapter.js";
 import { createSynologyAdapter } from "./synology/adapter.js";
 import { createUptimeKumaAdapter } from "./uptimekuma/adapter.js";
 import { createPhpIpamAdapter } from "./phpipam/adapter.js";
+import { createPbsAdapter } from "./pbs/adapter.js";
 
 export interface PingableAdapter {
   ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
@@ -38,6 +39,8 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
       return createUptimeKumaAdapter(config as any);
     case "phpipam":
       return createPhpIpamAdapter(config as any);
+    case "pbs":
+      return createPbsAdapter(config as any);
     default:
       throw new Error(`Integration type "${type}" is not implemented yet`);
   }

server/src/routes/integrations.ts

@@ -21,6 +21,7 @@ import { createSemaphoreAdapter } from "../integrations/semaphore/adapter.js";
 import { createProxmoxAdapter, guestsWithoutBackupCoverage } from "../integrations/proxmox/adapter.js";
 import { createSynologyAdapter } from "../integrations/synology/adapter.js";
 import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
+import { createPbsAdapter } from "../integrations/pbs/adapter.js";
 import { attachMatchedServers, summarizeMonitors, toMatchableServers } from "../services/uptimeKumaMatch.js";
 import { asyncHandler } from "../utils/asyncHandler.js";
 
@@ -805,3 +806,55 @@ integrationsRouter.get("/:id/uptimekuma/monitors", asyncHandler(async (req, res)
     res.status(502).json({ error: err instanceof Error ? err.message : String(err) });
   }
 }));
+
+// ─── Proxmox Backup Server ───────────────────────────────────────────────────
+// Read-only — no destructive actions (pruning/GC/deleting snapshots) are exposed.
+
+async function requirePbsAdapter(req: Request, res: Response) {
+  const id = Number(req.params.id);
+  const loaded = await loadIntegrationConfig(id);
+  if (!loaded) {
+    res.status(404).json({ error: "not_found" });
+    return null;
+  }
+  if (loaded.integration.type !== "pbs") {
+    res.status(400).json({ error: "wrong_type" });
+    return null;
+  }
+  if (!loaded.integration.enabled) {
+    res.status(400).json({ error: "integration_disabled" });
+    return null;
+  }
+  return { integration: loaded.integration, adapter: createPbsAdapter(loaded.config as any) };
+}
+
+integrationsRouter.get("/:id/pbs/datastores", asyncHandler(async (req, res) => {
+  const found = await requirePbsAdapter(req, res);
+  if (!found) return;
+
+  try {
+    const datastores = await found.adapter.listDatastores();
+    res.json({
+      datastores,
+      summary: {
+        datastoreCount: datastores.length,
+        failedSnapshotCount: datastores.reduce((sum, d) => sum + d.failedCount, 0),
+        unverifiedSnapshotCount: datastores.reduce((sum, d) => sum + d.unverifiedCount, 0),
+      },
+    });
+  } catch (err) {
+    res.status(502).json({ error: err instanceof Error ? err.message : String(err) });
+  }
+}));
+
+integrationsRouter.get("/:id/pbs/status", asyncHandler(async (req, res) => {
+  const found = await requirePbsAdapter(req, res);
+  if (!found) return;
+
+  try {
+    const status = await found.adapter.getNodeStatus();
+    res.json(status);
+  } catch (err) {
+    res.status(502).json({ error: err instanceof Error ? err.message : String(err) });
+  }
+}));

server/src/routes/settings.ts

@@ -7,6 +7,7 @@ import { scheduleSecretExpiryCheck } from "../services/secretExpiryScheduler.js"
 import { scheduleTailscaleKeyExpiryCheck } from "../services/tailscaleKeyExpiryScheduler.js";
 import { scheduleDockerUpdateCheck } from "../services/dockerUpdateScheduler.js";
 import { scheduleProxmoxBackupCheck } from "../services/proxmoxBackupScheduler.js";
+import { schedulePbsVerificationCheck } from "../services/pbsVerificationScheduler.js";
 import { scheduleLogRetentionPurge } from "../services/logRetentionScheduler.js";
 import { purgeOldLogs } from "../services/logRetention.js";
 import { scheduleQuietHoursFlush } from "../services/quietHoursScheduler.js";
@@ -68,6 +69,7 @@ const updateSchema = z.object({
       tailscaleKeyCheck: z.boolean(),
       dockerUpdateCheck: z.boolean(),
       proxmoxBackupCheck: z.boolean(),
+      pbsVerificationCheck: z.boolean(),
       healthAlerts: z.boolean(),
       automationAlerts: z.boolean(),
       domainExpiryCheck: z.boolean(),
@@ -111,6 +113,7 @@ settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
     await scheduleTailscaleKeyExpiryCheck();
     await scheduleDockerUpdateCheck();
     await scheduleProxmoxBackupCheck();
+    await schedulePbsVerificationCheck();
   }
   if (parsed.data.logRetention) {
     await scheduleLogRetentionPurge();

server/src/services/notify.ts

@@ -277,6 +277,22 @@ export async function notifyProxmoxUncoveredGuests(
   );
 }
 
+export async function notifyPbsVerificationFailed(
+  failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[],
+): Promise<void> {
+  if (failures.length === 0) return;
+  if (!(await eventEnabled("pbsVerificationCheck"))) return;
+  const lines = failures.map((f) =>
+    f.error
+      ? `${f.datastore} [${f.integrationName}]: couldn't be read — ${f.error}`
+      : `${f.datastore} [${f.integrationName}]: ${f.failedCount} snapshot${f.failedCount !== 1 ? "s" : ""} failed verification`,
+  );
+  await notify(
+    "Homelab Manager — Proxmox Backup Server Verification Failed",
+    `${failures.length} datastore${failures.length !== 1 ? "s have" : " has"} a problem:\n\n${lines.join("\n")}`,
+  );
+}
+
 export async function notifyHealthIssues(issues: { message: string }[]): Promise<void> {
   if (issues.length === 0) return;
   if (!(await eventEnabled("healthAlerts"))) return;

server/src/services/pbsVerificationScheduler.ts

@@ -0,0 +1,80 @@
+import schedule from "node-schedule";
+import { and, eq } from "drizzle-orm";
+import { db } from "../db/client.js";
+import { integrations } from "../db/schema.js";
+import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
+import { createPbsAdapter } from "../integrations/pbs/adapter.js";
+import { notifyPbsVerificationFailed } from "./notify.js";
+import { getSettings, getInternalFlag, setInternalFlag } from "./settingsStore.js";
+import { isInMaintenance } from "./maintenance.js";
+
+const LAST_RUN_FLAG = "pbsVerificationCheckLastRunDate";
+
+async function checkPbsVerification(): Promise<void> {
+  const rows = await db
+    .select({ id: integrations.id, name: integrations.name })
+    .from(integrations)
+    .where(and(eq(integrations.type, "pbs"), eq(integrations.enabled, true)));
+
+  const failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[] = [];
+
+  for (const row of rows) {
+    // A PBS host being worked on can't be reached reliably; this check is daily, so tomorrow's pass covers it.
+    if (await isInMaintenance("integration", row.id)) continue;
+    try {
+      const loaded = await loadIntegrationConfig(row.id);
+      if (!loaded) continue;
+      const adapter = createPbsAdapter(loaded.config as any);
+      const datastores = await adapter.listDatastores();
+
+      for (const d of datastores) {
+        if (d.error) {
+          failures.push({ integrationName: row.name, datastore: d.name, failedCount: 0, error: d.error });
+        } else if (d.failedCount > 0) {
+          failures.push({ integrationName: row.name, datastore: d.name, failedCount: d.failedCount, error: null });
+        }
+      }
+    } catch (err) {
+      console.error(`[pbsVerification] check failed for integration ${row.id}:`, err);
+    }
+  }
+
+  await notifyPbsVerificationFailed(failures);
+}
+
+async function checkPbsVerificationOnce(): Promise<void> {
+  const today = new Date().toDateString();
+  const lastRun = await getInternalFlag(LAST_RUN_FLAG);
+  if (lastRun === today) return;
+  await setInternalFlag(LAST_RUN_FLAG, today);
+  await checkPbsVerification();
+}
+
+function cronFromTime(time: string): string {
+  const [h, m] = time.split(":").map(Number);
+  return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} * * *`;
+}
+
+let currentJob: schedule.Job | null = null;
+
+/** (Re)schedules the daily PBS verification-failure check per the current notification settings. Call again after settings change. */
+export async function schedulePbsVerificationCheck(): Promise<void> {
+  if (currentJob) {
+    currentJob.cancel();
+    currentJob = null;
+  }
+  const { notifications } = await getSettings();
+  currentJob = schedule.scheduleJob({ rule: cronFromTime(notifications.secretCheckTime), tz: notifications.timezone }, () => {
+    setInternalFlag(LAST_RUN_FLAG, "").catch(() => {});
+    getSettings().then(({ notifications: n }) => {
+      if (n.pbsVerificationCheck) checkPbsVerification().catch((err) => console.error("[pbsVerification] check failed:", err));
+    });
+  });
+  console.log(`PBS verification check scheduled at ${notifications.secretCheckTime} (${notifications.timezone})`);
+}
+
+/** Runs once at startup (skipped if already run today), then arms the daily schedule. */
+export async function initPbsVerificationScheduler(): Promise<void> {
+  await checkPbsVerificationOnce();
+  await schedulePbsVerificationCheck();
+}

server/src/services/settingsStore.ts

@@ -42,6 +42,8 @@ export interface NotificationEvents {
   tailscaleKeyCheck: boolean;
   dockerUpdateCheck: boolean;
   proxmoxBackupCheck: boolean;
+  /** Daily reminder while a Proxmox Backup Server snapshot has failed verification, or a datastore couldn't be read at all. */
+  pbsVerificationCheck: boolean;
   healthAlerts: boolean;
   /** A Semaphore template or Gitea repo whose latest run failed, and when it succeeds again. */
   automationAlerts: boolean;
@@ -125,6 +127,7 @@ const DEFAULTS: AppSettings = {
     tailscaleKeyCheck: true,
     dockerUpdateCheck: true,
     proxmoxBackupCheck: true,
+    pbsVerificationCheck: true,
     healthAlerts: true,
     automationAlerts: true,
     domainExpiryCheck: true,