Acknowledge and hand over alerts, incident timelines, and built-in database backups

- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what
  needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics
- Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents,
  incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs,
  related alerts and notes; Incidents page and a digest section
- Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download,
  check, upload, delete, and restore staged and applied at start-up before the database opens
  (pre-restore copy kept); alerts and /metrics for stale or failing backups
- Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 02:21:08 +02:00
1 parent 7a5b623444
commit 2744a84719
41 files changed
+6573 -26

No files matched your search

+21
View File
@@ -20,6 +20,8 @@ import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
import { attentionCounts, loadAcks } from "./alertAcks.js";
import { listIncidents } from "./incidents.js";
export const DIGEST_DAYS = 7;
const DAY_MS = 24 * 3600_000;
@@ -90,11 +92,30 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
if (report.counts.silenced > 0) held.push(`${report.counts.silenced} silenced by maintenance`);
if (report.counts.grouped > 0) held.push(`${report.counts.grouped} folded under a server that is down`);
if (held.length > 0) lines.push(`(${held.join(", ")})`);
const acknowledged = attentionCounts(report.alerts, await loadAcks(now)).acknowledged;
if (acknowledged > 0) lines.push(`(${acknowledged} of them acknowledged — somebody is on it)`);
sections.push({ heading: "Right now", lines, attention: active.length > 0 || report.couldntCheck.length > 0 });
} catch (err) {
console.error("[digest] couldn't read the alerts:", err);
}
// ── Incidents ──
try {
const since = now.getTime() - DIGEST_DAYS * DAY_MS;
const all = await listIncidents({ now });
const begun = all.filter((i) => Date.parse(i.openedAt) >= since);
const ended = all.filter((i) => i.resolvedAt !== null && Date.parse(i.resolvedAt) >= since);
const open = all.filter((i) => i.open);
if (begun.length > 0 || ended.length > 0) {
const lines = [`${plural(begun.length, "incident")} began, ${ended.length} ended, ${open.length} still open.`];
const longest = [...ended].sort((a, b) => b.durationMs - a.durationMs).slice(0, 3);
lines.push(...longest.map((i) => `- ${i.message} — lasted ${spanWords(i.durationMs / 60_000)}`));
sections.push({ heading: "Incidents", lines, attention: open.length > 0 });
}
} catch (err) {
console.error("[digest] couldn't read the incidents:", err);
}
// ── Uptime ──
try {
const uptime = await getUptime(now, DIGEST_DAYS);