Stale-backup alerts, per-server alert limits, and update runs from the Updates page
- Backups: a guest whose newest backup is older than its limit is an alert (backup kind), in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds) - Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent, forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the global ones were; saving clears the alerts cache - Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and reports what is left and whether a restart is due; poller survives restarts - Notification toggle for update-run results; tests and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5dc6cea69c
commit
7a5b623444
42 files changed
+7561
-57
No files matched your search
@@ -11,6 +11,8 @@ import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
|
||||
import { getSourceStatus } from "./integrationHealthMonitor.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
|
||||
import { parseThresholds } from "./serverThresholds.js";
|
||||
import { getCachedBackupCoverage } from "./backupCoverage.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
@@ -58,8 +60,9 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
// ── Servers ──
|
||||
const latest = latestAgentVersions();
|
||||
const serverRows = await db.select().from(servers);
|
||||
const offlineAfterMs = settings.healthChecks.serverOfflineMinutes * 60_000;
|
||||
for (const s of serverRows) {
|
||||
// A server may have its own "offline after"; otherwise the global one.
|
||||
const offlineAfterMs = (parseThresholds(s.thresholds)?.serverOfflineMinutes ?? settings.healthChecks.serverOfflineMinutes) * 60_000;
|
||||
if (!s.lastSeenAt) continue; // never reported: nothing to say about it
|
||||
const server = s.name;
|
||||
const seen = Date.parse(s.lastSeenAt);
|
||||
@@ -115,6 +118,14 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
// ── Alerts ──
|
||||
try {
|
||||
const report = await getAlerts(false);
|
||||
// What the backup check last found (the alert pass above refreshes it); never reads Proxmox or Backup Server itself.
|
||||
const coverage = await getCachedBackupCoverage(now);
|
||||
if (coverage) {
|
||||
for (const status of ["ok", "stale", "failed", "none", "unknown"] as const) out.add("homelab_backup_guests", "gauge", "Proxmox guests by how well they are backed up (ok, stale, failed, none, unknown).", coverage.counts[status], { status });
|
||||
for (const g of coverage.guests) {
|
||||
if (g.lastBackupAt) out.add("homelab_backup_last_age_seconds", "gauge", "How long ago a guest's newest backup was.", Math.max(0, Math.floor((now.getTime() - Date.parse(g.lastBackupAt)) / 1000)), { guest: g.name, vmid: String(g.vmid), integration: g.integrationName });
|
||||
}
|
||||
}
|
||||
for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity });
|
||||
out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced);
|
||||
out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped);
|
||||
|
||||
Reference in new issue
Block a user