Acknowledge and hand over alerts, incident timelines, and built-in database backups

- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what
  needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics
- Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents,
  incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs,
  related alerts and notes; Incidents page and a digest section
- Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download,
  check, upload, delete, and restore staged and applied at start-up before the database opens
  (pre-restore copy kept); alerts and /metrics for stale or failing backups
- Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 02:21:08 +02:00
1 parent 7a5b623444
commit 2744a84719
41 files changed
+6573 -26

No files matched your search

@@ -0,0 +1,65 @@
import schedule from "node-schedule";
import { getSettings } from "./settingsStore.js";
import { createSnapshot, lastAttempt, newestSnapshot, pruneSnapshots } from "./selfBackup.js";
import { notify } from "./notify.js";
let currentJob: schedule.Job | null = null;
/** The cron rule for "every day at HH:MM". */
export function cronForBackup(time: string): string {
const [h, m] = time.split(":").map(Number);
return `${Number.isFinite(m) ? m : 30} ${Number.isFinite(h) ? h : 3} * * *`;
}
/** One scheduled backup: take it, then drop the oldest beyond how many are kept. Says so through the channels if it fails. */
export async function runScheduledBackup(now: Date = new Date()): Promise<{ name: string } | null> {
const { selfBackup, notifications } = await getSettings();
try {
const { info } = await createSnapshot("scheduled", now);
pruneSnapshots(selfBackup.keep);
return { name: info.name };
} catch (err) {
console.error("[selfBackup] the scheduled backup failed:", err instanceof Error ? err.message : err);
if (notifications.healthAlerts !== false) {
await notify("Homelab Manager — its own backup failed", `The scheduled backup of Homelab Manager's database didn't work: ${err instanceof Error ? err.message : String(err)}`).catch(() => {});
}
return null;
}
}
/** The next time it will run, or null when it is off. */
export function nextBackupRun(): Date | null {
const next = currentJob?.nextInvocation();
return next ? new Date(next.getTime()) : null;
}
/** (Re)schedules the daily backup per the current settings. Call again after they change. */
export async function scheduleSelfBackup(): Promise<void> {
if (currentJob) {
currentJob.cancel();
currentJob = null;
}
const { selfBackup, notifications } = await getSettings();
if (!selfBackup.enabled) {
console.log("[selfBackup] daily backups are off, nothing scheduled");
return;
}
currentJob = schedule.scheduleJob({ rule: cronForBackup(selfBackup.time), tz: notifications.timezone }, () => {
void runScheduledBackup();
});
console.log(`Daily backup scheduled at ${selfBackup.time} (${notifications.timezone})`);
}
/** Schedules it, and takes one soon if the app has been off through its time (or has never taken one). */
export async function initSelfBackupScheduler(): Promise<void> {
await scheduleSelfBackup();
const { selfBackup } = await getSettings();
if (!selfBackup.enabled) return;
const newest = newestSnapshot();
const overdue = !newest || Date.now() - Date.parse(newest.at) > 26 * 3600_000;
// Only retry a failing one at the next scheduled time, not at every restart.
const last = await lastAttempt();
if (overdue && !(last && !last.ok && Date.now() - Date.parse(last.at) < 6 * 3600_000)) {
setTimeout(() => void runScheduledBackup(), 30_000).unref();
}
}