Files
Homelab-manager/server/src/services/proxmoxBackupScheduler.ts
T
bobbanandClaude Sonnet 5.5 447f33fff6 Add an Alerts page under Operations listing everything that's wrong now
One list of the current problems across servers and integrations, instead
of waiting for a notification or visiting each page: servers that stopped
reporting, full or nearly full disks and volumes (critical from 95%),
Synology volume/disk problems, failed or uncovered Proxmox backups,
failed Proxmox Backup Server verifications, container image updates,
expired or expiring secrets/domains/Tailscale keys, failed Semaphore and
Gitea runs, Uptime Kuma monitors that are down, overdue osTicket tickets,
and integrations whose calls keep failing. Visible to every role, with
severity and kind filters, search, sorting, CSV export and "Check now".

It runs the same checks that send the notifications rather than a second
copy of them: the detection in the health, automation, Proxmox backup,
PBS, Docker update and Tailscale key checks is pulled out into shared
collectors that both the schedulers and the page call, so the two can't
disagree about what counts as a problem. Notification behaviour is
unchanged, including the scheduled backup checks skipping integrations
under a maintenance window. Unlike the notifications the page ignores the
on/off toggles, and keeps problems under a maintenance window, marked
silenced and counted apart.

It reads live, so a result is reused for a minute (and Refresh can't
re-run everything more than once every ten seconds), and every source has
a 20 s limit so one hung integration can't hang the page. Anything it
couldn't read is called out at the top instead of looking like all clear,
and server checks pause for the same 20 minutes after a restart as the
notifications do, with a note saying so.

Also gives the newer integrations (PBS, osTicket, Uptime Kuma, phpIPAM)
proper names in "integration down" notifications instead of their ids.

Verified through the real routes against a scratch database with fake
backends (offline and full-disk servers, secrets and domains, a silenced
server, a fake PBS with failed verification, a hanging integration, a
refused one, a failing-calls streak, caching, the restart grace period,
auth), and by rendering the real page against that data in a browser:
filters, search, silenced toggle, sorting, Check now, dark mode.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-02 23:07:57 +02:00

133 lines
5.4 KiB
TypeScript

import schedule from "node-schedule";
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, guestsWithoutBackupCoverage, type ProxmoxBackupTask } from "../integrations/proxmox/adapter.js";
import { notifyProxmoxBackupFailure, notifyProxmoxUncoveredGuests } from "./notify.js";
import { getSettings, getInternalFlag, setInternalFlag } from "./settingsStore.js";
import { activeSubjects } from "./maintenance.js";
import type { SourceFailure } from "./alertTypes.js";
const LAST_RUN_FLAG = "proxmoxBackupCheckLastRunDate";
export interface BackupFailure {
integrationId: number;
integrationName: string;
node: string;
guestId: string | null;
status: string;
silenced: boolean;
}
export interface UncoveredGuest {
integrationId: number;
integrationName: string;
guestName: string;
vmid: number;
node: string;
silenced: boolean;
}
/**
* Failed latest backups and guests no backup job covers, across the enabled Proxmox integrations. The daily check skips
* integrations under a maintenance window altogether (a host being worked on can't run or report backups, and tomorrow's
* pass covers it); the Alerts page passes skipSilenced: false to see them, flagged.
*/
export async function collectProxmoxBackupProblems(opts: {
skipSilenced: boolean;
}): Promise<{ failures: BackupFailure[]; uncovered: UncoveredGuest[]; sourceFailures: SourceFailure[] }> {
const rows = await db
.select({ id: integrations.id, name: integrations.name })
.from(integrations)
.where(and(eq(integrations.type, "proxmox"), eq(integrations.enabled, true)));
const failures: BackupFailure[] = [];
const uncovered: UncoveredGuest[] = [];
const sourceFailures: SourceFailure[] = [];
const silencedSubjects = await activeSubjects();
for (const row of rows) {
const silenced = silencedSubjects.has(`integration:${row.id}`);
if (silenced && opts.skipSilenced) continue;
try {
const loaded = await loadIntegrationConfig(row.id);
if (!loaded) continue;
const adapter = createProxmoxAdapter(loaded.config as any);
const [tasks, jobs, guests] = await Promise.all([
adapter.listRecentBackupTasks(),
adapter.listBackupJobs(),
adapter.listGuests(),
]);
// Each node runs its own backup schedule independently, so check the
// most recent run per node rather than only the single most recent
// task overall — otherwise a failing node could be masked by a
// healthier node's more recent run.
const latestByNode = new Map<string, ProxmoxBackupTask>();
for (const t of tasks) {
const existing = latestByNode.get(t.node);
if (!existing || t.startTime > existing.startTime) latestByNode.set(t.node, t);
}
for (const [node, task] of latestByNode) {
if (!task.ok && task.status !== "running") {
failures.push({ integrationId: row.id, integrationName: row.name, node, guestId: task.guestId, status: task.status, silenced });
}
}
for (const g of guestsWithoutBackupCoverage(guests, jobs)) {
uncovered.push({ integrationId: row.id, integrationName: row.name, guestName: g.name, vmid: g.vmid, node: g.node, silenced });
}
} catch (err) {
console.error(`[proxmoxBackup] check failed for integration ${row.id}:`, err);
sourceFailures.push({ integrationId: row.id, integrationName: row.name, message: err instanceof Error ? err.message : String(err) });
}
}
return { failures, uncovered, sourceFailures };
}
async function checkProxmoxBackups(): Promise<void> {
const { failures, uncovered } = await collectProxmoxBackupProblems({ skipSilenced: true });
await notifyProxmoxBackupFailure(failures);
await notifyProxmoxUncoveredGuests(uncovered);
}
async function checkProxmoxBackupsOnce(): Promise<void> {
const today = new Date().toDateString();
const lastRun = await getInternalFlag(LAST_RUN_FLAG);
if (lastRun === today) return;
await setInternalFlag(LAST_RUN_FLAG, today);
await checkProxmoxBackups();
}
function cronFromTime(time: string): string {
const [h, m] = time.split(":").map(Number);
return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} * * *`;
}
let currentJob: schedule.Job | null = null;
/** (Re)schedules the daily Proxmox backup-failure check per the current notification settings. Call again after settings change. */
export async function scheduleProxmoxBackupCheck(): Promise<void> {
if (currentJob) {
currentJob.cancel();
currentJob = null;
}
const { notifications } = await getSettings();
currentJob = schedule.scheduleJob({ rule: cronFromTime(notifications.secretCheckTime), tz: notifications.timezone }, () => {
setInternalFlag(LAST_RUN_FLAG, "").catch(() => {});
getSettings().then(({ notifications: n }) => {
if (n.proxmoxBackupCheck) checkProxmoxBackups().catch((err) => console.error("[proxmoxBackup] check failed:", err));
});
});
console.log(`Proxmox backup check scheduled at ${notifications.secretCheckTime} (${notifications.timezone})`);
}
/** Runs once at startup (skipped if already run today), then arms the daily schedule. */
export async function initProxmoxBackupScheduler(): Promise<void> {
await checkProxmoxBackupsOnce();
await scheduleProxmoxBackupCheck();
}