import schedule from "node-schedule"; import { and, eq } from "drizzle-orm"; import { db } from "../db/client.js"; import { integrations } from "../db/schema.js"; import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; import { createProxmoxAdapter, guestsWithoutBackupCoverage, type ProxmoxBackupTask } from "../integrations/proxmox/adapter.js"; import { notifyProxmoxBackupFailure, notifyProxmoxUncoveredGuests } from "./notify.js"; import { getSettings, getInternalFlag, setInternalFlag } from "./settingsStore.js"; import { activeSubjects } from "./maintenance.js"; import type { SourceFailure } from "./alertTypes.js"; const LAST_RUN_FLAG = "proxmoxBackupCheckLastRunDate"; export interface BackupFailure { integrationId: number; integrationName: string; node: string; guestId: string | null; status: string; silenced: boolean; } export interface UncoveredGuest { integrationId: number; integrationName: string; guestName: string; vmid: number; node: string; silenced: boolean; } /** * Failed latest backups and guests no backup job covers, across the enabled Proxmox integrations. The daily check skips * integrations under a maintenance window altogether (a host being worked on can't run or report backups, and tomorrow's * pass covers it); the Alerts page passes skipSilenced: false to see them, flagged. */ export async function collectProxmoxBackupProblems(opts: { skipSilenced: boolean; }): Promise<{ failures: BackupFailure[]; uncovered: UncoveredGuest[]; sourceFailures: SourceFailure[] }> { const rows = await db .select({ id: integrations.id, name: integrations.name }) .from(integrations) .where(and(eq(integrations.type, "proxmox"), eq(integrations.enabled, true))); const failures: BackupFailure[] = []; const uncovered: UncoveredGuest[] = []; const sourceFailures: SourceFailure[] = []; const silencedSubjects = await activeSubjects(); for (const row of rows) { const silenced = silencedSubjects.has(`integration:${row.id}`); if (silenced && opts.skipSilenced) continue; try { const loaded = await loadIntegrationConfig(row.id); if (!loaded) continue; const adapter = createProxmoxAdapter(loaded.config as any); const [tasks, jobs, guests] = await Promise.all([ adapter.listRecentBackupTasks(), adapter.listBackupJobs(), adapter.listGuests(), ]); // Each node runs its own backup schedule independently, so check the // most recent run per node rather than only the single most recent // task overall — otherwise a failing node could be masked by a // healthier node's more recent run. const latestByNode = new Map(); for (const t of tasks) { const existing = latestByNode.get(t.node); if (!existing || t.startTime > existing.startTime) latestByNode.set(t.node, t); } for (const [node, task] of latestByNode) { if (!task.ok && task.status !== "running") { failures.push({ integrationId: row.id, integrationName: row.name, node, guestId: task.guestId, status: task.status, silenced }); } } for (const g of guestsWithoutBackupCoverage(guests, jobs)) { uncovered.push({ integrationId: row.id, integrationName: row.name, guestName: g.name, vmid: g.vmid, node: g.node, silenced }); } } catch (err) { console.error(`[proxmoxBackup] check failed for integration ${row.id}:`, err); sourceFailures.push({ integrationId: row.id, integrationName: row.name, message: err instanceof Error ? err.message : String(err) }); } } return { failures, uncovered, sourceFailures }; } async function checkProxmoxBackups(): Promise { const { failures, uncovered } = await collectProxmoxBackupProblems({ skipSilenced: true }); await notifyProxmoxBackupFailure(failures); await notifyProxmoxUncoveredGuests(uncovered); } async function checkProxmoxBackupsOnce(): Promise { const today = new Date().toDateString(); const lastRun = await getInternalFlag(LAST_RUN_FLAG); if (lastRun === today) return; await setInternalFlag(LAST_RUN_FLAG, today); await checkProxmoxBackups(); } function cronFromTime(time: string): string { const [h, m] = time.split(":").map(Number); return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} * * *`; } let currentJob: schedule.Job | null = null; /** (Re)schedules the daily Proxmox backup-failure check per the current notification settings. Call again after settings change. */ export async function scheduleProxmoxBackupCheck(): Promise { if (currentJob) { currentJob.cancel(); currentJob = null; } const { notifications } = await getSettings(); currentJob = schedule.scheduleJob({ rule: cronFromTime(notifications.secretCheckTime), tz: notifications.timezone }, () => { setInternalFlag(LAST_RUN_FLAG, "").catch(() => {}); getSettings().then(({ notifications: n }) => { if (n.proxmoxBackupCheck) checkProxmoxBackups().catch((err) => console.error("[proxmoxBackup] check failed:", err)); }); }); console.log(`Proxmox backup check scheduled at ${notifications.secretCheckTime} (${notifications.timezone})`); } /** Runs once at startup (skipped if already run today), then arms the daily schedule. */ export async function initProxmoxBackupScheduler(): Promise { await checkProxmoxBackupsOnce(); await scheduleProxmoxBackupCheck(); }