Acknowledge and hand over alerts, incident timelines, and built-in database backups
- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics - Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents, incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs, related alerts and notes; Incidents page and a digest section - Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download, check, upload, delete, and restore staged and applied at start-up before the database opens (pre-restore copy kept); alerts and /metrics for stale or failing backups - Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7a5b623444
commit
2744a84719
41 files changed
+6573
-26
No files matched your search
@@ -15,6 +15,9 @@ import { integrations, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
import { recordFromReport } from "./incidents.js";
|
||||
import { reconcileAcks } from "./alertAcks.js";
|
||||
import { lastAttempt as lastSelfBackup, newestSnapshot, STALE_AFTER_HOURS as SELF_BACKUP_STALE_HOURS } from "./selfBackup.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
|
||||
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
|
||||
@@ -53,6 +56,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
gitea: { label: "Gitea", link: "/gitea" },
|
||||
dockhand: { label: "Docker", link: "/docker" },
|
||||
tailscale: { label: "Tailscale", link: "/tailscale" },
|
||||
homelab: { label: "Homelab Manager", link: "/settings/backup" },
|
||||
pbs: { label: "Proxmox Backup", link: "/pbs" },
|
||||
uptimekuma: { label: "Uptime Kuma", link: "/uptime-kuma" },
|
||||
osticket: { label: "osTicket", link: "/osticket" },
|
||||
@@ -262,6 +266,29 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
}
|
||||
}),
|
||||
|
||||
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
|
||||
// stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it.
|
||||
check("Homelab Manager's own backups", async () => {
|
||||
const { selfBackup } = await getSettings();
|
||||
if (!selfBackup.enabled) return;
|
||||
const newest = newestSnapshot();
|
||||
const last = await lastSelfBackup();
|
||||
const failing = last && !last.ok ? ` The last attempt failed: ${last.error}` : "";
|
||||
if (!newest) {
|
||||
if (failing) push({ category: "backup", severity: "warning", type: "homelab", message: `Homelab Manager has no backup of its own yet.${failing}`, key: "self-backup" });
|
||||
return;
|
||||
}
|
||||
const ageHours = (Date.now() - Date.parse(newest.at)) / 3600_000;
|
||||
if (ageHours <= SELF_BACKUP_STALE_HOURS) return;
|
||||
push({
|
||||
category: "backup",
|
||||
severity: ageHours >= 24 * 7 ? "critical" : "warning",
|
||||
type: "homelab",
|
||||
message: `The newest backup of Homelab Manager's own database is ${ageHours < 72 ? `${Math.round(ageHours)} h` : `${Math.floor(ageHours / 24)} days`} old.${failing}`,
|
||||
key: "self-backup",
|
||||
});
|
||||
}),
|
||||
|
||||
check("Backup verification", async () => {
|
||||
const { problems, sourceFailures } = await collectPbsProblems({ skipSilenced: false });
|
||||
for (const p of problems) {
|
||||
@@ -475,6 +502,8 @@ export async function getAlerts(force: boolean): Promise<AlertsReport & { cached
|
||||
inFlight ??= collectAlerts()
|
||||
.then((report) => {
|
||||
cache = { at: Date.now(), report };
|
||||
// Remembered for the incident record and the acknowledgements — in the background, and never allowed to hold up or break the answer.
|
||||
void recordFromReport(report).then(() => reconcileAcks(new Set(report.alerts.map((a) => a.id)))).catch((err) => console.error("[alerts] couldn't update the incident record:", err));
|
||||
return report;
|
||||
})
|
||||
.finally(() => {
|
||||
|
||||
Reference in new issue
Block a user