Stale-backup alerts, per-server alert limits, and update runs from the Updates page

- Backups: a guest whose newest backup is older than its limit is an alert (backup kind),
  in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds)
- Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent,
  forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the
  global ones were; saving clears the alerts cache
- Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under
  maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and
  reports what is left and whether a restart is due; poller survives restarts
- Notification toggle for update-run results; tests and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 01:25:09 +02:00
1 parent 5dc6cea69c
commit 7a5b623444
42 files changed
+7561 -57

No files matched your search

+27 -3
View File
@@ -13,6 +13,8 @@ import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
@@ -241,6 +243,25 @@ export async function collectAlerts(): Promise<AlertsReport> {
sourceFailures.forEach((f) => cannotCheckIntegration(f, "Proxmox backups"));
}),
// A guest whose newest backup is older than its limit — which Proxmox's own "latest run failed" alert can't see, because that only
// looks at the most recent run per node.
check("Backup coverage", async () => {
const now = new Date();
const report = await getBackupCoverage({ now, rules: true });
for (const s of staleBackups(report, now, subjects)) {
push({
category: "backup",
severity: s.severity,
type: "proxmox",
message: s.message,
key: s.key,
link: "/backups",
silenced: s.silenced,
target: s.guest.server ? targetOf("server", s.guest.server.id) : integrationTarget(s.guest.integrationId),
});
}
}),
check("Backup verification", async () => {
const { problems, sourceFailures } = await collectPbsProblems({ skipSilenced: false });
for (const p of problems) {
@@ -376,10 +397,13 @@ export async function collectAlerts(): Promise<AlertsReport> {
// Not a notification: a disk that is on course to fill is something to see coming, so it is shown here, from the server's own history.
check("Disk forecasts", async () => {
const { healthChecks } = await getSettings();
if (!(healthChecks.diskForecastDays > 0)) return;
for (const s of await db.select({ id: servers.id, name: servers.name }).from(servers)) {
for (const s of await db.select({ id: servers.id, name: servers.name, thresholds: servers.thresholds }).from(servers)) {
// A server may have its own forecast window (0 turns it off for that server), its own limit, and disks it is told to leave alone.
const own = parseThresholds(s.thresholds);
const forecastDays = effectiveThresholds(healthChecks, own).diskForecastDays;
if (!(forecastDays > 0)) continue;
const series = await getServerMetrics(s.id, 14);
for (const f of disksFillingSoon(series, healthChecks.diskForecastDays, healthChecks.diskUsagePercent)) {
for (const f of disksFillingSoon(series, forecastDays, (mount) => diskLimit(own, mount, healthChecks.diskUsagePercent))) {
push({
category: "disk",
severity: f.daysToFull <= 3 ? "critical" : "warning",