Acknowledge and hand over alerts, incident timelines, and built-in database backups

- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what
  needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics
- Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents,
  incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs,
  related alerts and notes; Incidents page and a digest section
- Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download,
  check, upload, delete, and restore staged and applied at start-up before the database opens
  (pre-restore copy kept); alerts and /metrics for stale or failing backups
- Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 02:21:08 +02:00
1 parent 7a5b623444
commit 2744a84719
41 files changed
+6573 -26

No files matched your search

+18
View File
@@ -13,6 +13,9 @@ import { parsePendingUpdates } from "./pendingUpdates.js";
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
import { parseThresholds } from "./serverThresholds.js";
import { getCachedBackupCoverage } from "./backupCoverage.js";
import { lastAttempt as lastSelfBackup, listSnapshots } from "./selfBackup.js";
import { loadAcks, attentionCounts } from "./alertAcks.js";
import { listIncidents } from "./incidents.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { computeSecretStatus } from "./secretStatus.js";
import { getSettings } from "./settingsStore.js";
@@ -115,6 +118,18 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
console.error("[metrics] couldn't read the uptime:", err);
}
// ── Homelab Manager's own backups ──
try {
const snapshots = listSnapshots();
out.add("homelab_selfbackup_snapshots", "gauge", "How many backups of Homelab Manager's own database are kept.", snapshots.length);
const newest = snapshots.find((s) => s.kind === "scheduled" || s.kind === "manual");
if (newest) out.add("homelab_selfbackup_last_success_timestamp_seconds", "gauge", "When the newest backup of Homelab Manager's own database was made, as a Unix time.", Math.floor(Date.parse(newest.at) / 1000));
const last = await lastSelfBackup();
if (last) out.add("homelab_selfbackup_last_attempt_ok", "gauge", "1 if the last attempt to back up Homelab Manager's own database worked, 0 if it failed.", last.ok ? 1 : 0);
} catch (err) {
console.error("[metrics] couldn't read the backups folder:", err);
}
// ── Alerts ──
try {
const report = await getAlerts(false);
@@ -127,6 +142,9 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
}
}
for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity });
// Somebody has said they are dealing with these (still raised, and still counted above).
out.add("homelab_alerts_acknowledged", "gauge", "Alerts somebody has acknowledged and that are not silenced by maintenance.", attentionCounts(report.alerts, await loadAcks(now)).acknowledged);
out.add("homelab_incidents_open", "gauge", "Incidents that have begun and not yet ended.", (await listIncidents({ limit: 1, now })).filter((i) => i.open).length);
out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced);
out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped);
} catch (err) {