Stale-backup alerts, per-server alert limits, and update runs from the Updates page

- Backups: a guest whose newest backup is older than its limit is an alert (backup kind),
  in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds)
- Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent,
  forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the
  global ones were; saving clears the alerts cache
- Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under
  maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and
  reports what is left and whether a restart is due; poller survives restarts
- Notification toggle for update-run results; tests and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 01:25:09 +02:00
1 parent 5dc6cea69c
commit 7a5b623444
42 files changed
+7561 -57

No files matched your search

@@ -54,6 +54,7 @@ const DEFAULT_NOTIFICATIONS: NotificationEvents = {
pbsVerificationCheck: true,
healthAlerts: true,
automationAlerts: true,
updateRunResults: true,
domainExpiryCheck: true,
secretCheckTime: "08:00",
timezone: "UTC",
@@ -72,7 +73,7 @@ const DIGEST_DAYS: { value: number; label: string }[] = [
{ value: 6, label: "Saturday" },
{ value: 0, label: "Sunday" },
];
const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 };
const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 };
type TestResult = { ok: boolean; message: string } | null;
type EventKey = { [K in keyof NotificationEvents]: NotificationEvents[K] extends boolean ? K : never }[keyof NotificationEvents];
@@ -688,7 +689,16 @@ export default function NotificationSettings() {
<h3 className="card-title">Backups and updates</h3>
</div>
<div className="card-body">
{toggle("proxmoxBackupCheck", "Proxmox backup failed or a guest has no coverage")}
{toggle("proxmoxBackupCheck", "Proxmox backup failed, a guest has no coverage, or its newest backup is too old")}
<NumberField
label="A guest's backup is too old after"
value={healthChecks.backupStaleHours}
min={0}
max={2160}
unit="hours"
hint="Shown on the Alerts page for every guest whose newest backup (on Proxmox or a Backup Server) is older than this, and sent once a day with the other backup messages. A server can have its own limit. 0 turns it off."
onChange={(v) => setHealthChecks((h) => ({ ...h, backupStaleHours: v }))}
/>
{toggle("pbsVerificationCheck", "Proxmox Backup Server snapshot failed verification")}
{toggle("dockerUpdateCheck", "Docker image update available")}
</div>
@@ -713,6 +723,7 @@ export default function NotificationSettings() {
onChange={(v) => setNotifications((n) => ({ ...n, integrationFailureThreshold: v }))}
/>
<div className="mt-3">{toggle("automationAlerts", "Semaphore template or Gitea workflow run failed")}</div>
<div className="mt-3">{toggle("updateRunResults", "An update run you started finished, or didn't")}</div>
</div>
</div>
</div>