Acknowledge and hand over alerts, incident timelines, and built-in database backups
- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics - Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents, incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs, related alerts and notes; Incidents page and a digest section - Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download, check, upload, delete, and restore staged and applied at start-up before the database opens (pre-restore copy kept); alerts and /metrics for stale or failing backups - Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7a5b623444
commit
2744a84719
41 files changed
+6573
-26
No files matched your search
@@ -20,6 +20,8 @@ import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
import { attentionCounts, loadAcks } from "./alertAcks.js";
|
||||
import { listIncidents } from "./incidents.js";
|
||||
|
||||
export const DIGEST_DAYS = 7;
|
||||
const DAY_MS = 24 * 3600_000;
|
||||
@@ -90,11 +92,30 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
if (report.counts.silenced > 0) held.push(`${report.counts.silenced} silenced by maintenance`);
|
||||
if (report.counts.grouped > 0) held.push(`${report.counts.grouped} folded under a server that is down`);
|
||||
if (held.length > 0) lines.push(`(${held.join(", ")})`);
|
||||
const acknowledged = attentionCounts(report.alerts, await loadAcks(now)).acknowledged;
|
||||
if (acknowledged > 0) lines.push(`(${acknowledged} of them acknowledged — somebody is on it)`);
|
||||
sections.push({ heading: "Right now", lines, attention: active.length > 0 || report.couldntCheck.length > 0 });
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the alerts:", err);
|
||||
}
|
||||
|
||||
// ── Incidents ──
|
||||
try {
|
||||
const since = now.getTime() - DIGEST_DAYS * DAY_MS;
|
||||
const all = await listIncidents({ now });
|
||||
const begun = all.filter((i) => Date.parse(i.openedAt) >= since);
|
||||
const ended = all.filter((i) => i.resolvedAt !== null && Date.parse(i.resolvedAt) >= since);
|
||||
const open = all.filter((i) => i.open);
|
||||
if (begun.length > 0 || ended.length > 0) {
|
||||
const lines = [`${plural(begun.length, "incident")} began, ${ended.length} ended, ${open.length} still open.`];
|
||||
const longest = [...ended].sort((a, b) => b.durationMs - a.durationMs).slice(0, 3);
|
||||
lines.push(...longest.map((i) => `- ${i.message} — lasted ${spanWords(i.durationMs / 60_000)}`));
|
||||
sections.push({ heading: "Incidents", lines, attention: open.length > 0 });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the incidents:", err);
|
||||
}
|
||||
|
||||
// ── Uptime ──
|
||||
try {
|
||||
const uptime = await getUptime(now, DIGEST_DAYS);
|
||||
|
||||
Reference in new issue
Block a user