Stale-backup alerts, per-server alert limits, and update runs from the Updates page

- Backups: a guest whose newest backup is older than its limit is an alert (backup kind),
  in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds)
- Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent,
  forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the
  global ones were; saving clears the alerts cache
- Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under
  maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and
  reports what is left and whether a restart is due; poller survives restarts
- Notification toggle for update-run results; tests and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 01:25:09 +02:00
1 parent 5dc6cea69c
commit 7a5b623444
42 files changed
+7561 -57

No files matched your search

+12 -1
View File
@@ -11,6 +11,8 @@ import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
import { getSourceStatus } from "./integrationHealthMonitor.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
import { parseThresholds } from "./serverThresholds.js";
import { getCachedBackupCoverage } from "./backupCoverage.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { computeSecretStatus } from "./secretStatus.js";
import { getSettings } from "./settingsStore.js";
@@ -58,8 +60,9 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
// ── Servers ──
const latest = latestAgentVersions();
const serverRows = await db.select().from(servers);
const offlineAfterMs = settings.healthChecks.serverOfflineMinutes * 60_000;
for (const s of serverRows) {
// A server may have its own "offline after"; otherwise the global one.
const offlineAfterMs = (parseThresholds(s.thresholds)?.serverOfflineMinutes ?? settings.healthChecks.serverOfflineMinutes) * 60_000;
if (!s.lastSeenAt) continue; // never reported: nothing to say about it
const server = s.name;
const seen = Date.parse(s.lastSeenAt);
@@ -115,6 +118,14 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
// ── Alerts ──
try {
const report = await getAlerts(false);
// What the backup check last found (the alert pass above refreshes it); never reads Proxmox or Backup Server itself.
const coverage = await getCachedBackupCoverage(now);
if (coverage) {
for (const status of ["ok", "stale", "failed", "none", "unknown"] as const) out.add("homelab_backup_guests", "gauge", "Proxmox guests by how well they are backed up (ok, stale, failed, none, unknown).", coverage.counts[status], { status });
for (const g of coverage.guests) {
if (g.lastBackupAt) out.add("homelab_backup_last_age_seconds", "gauge", "How long ago a guest's newest backup was.", Math.max(0, Math.floor((now.getTime() - Date.parse(g.lastBackupAt)) / 1000)), { guest: g.name, vmid: String(g.vmid), integration: g.integrationName });
}
}
for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity });
out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced);
out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped);