Acknowledge and hand over alerts, incident timelines, and built-in database backups

- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what
  needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics
- Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents,
  incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs,
  related alerts and notes; Incidents page and a digest section
- Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download,
  check, upload, delete, and restore staged and applied at start-up before the database opens
  (pre-restore copy kept); alerts and /metrics for stale or failing backups
- Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 02:21:08 +02:00
1 parent 7a5b623444
commit 2744a84719
41 files changed
+6573 -26

No files matched your search

+69 -2
View File
@@ -1,6 +1,10 @@
import { Router } from "express";
import { requireAuth } from "../auth/middleware.js";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { getAlerts } from "../services/alerts.js";
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
import { recordAudit } from "../services/audit.js";
import { openIncidentIds } from "../services/incidents.js";
import { ticketsForAlerts, type AlertTicketInfo } from "../services/alertTickets.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -19,5 +23,68 @@ alertsRouter.get("/", asyncHandler(async (req, res) => {
} catch (err) {
console.error("[alerts] couldn't look up tickets:", err);
}
res.json({ ...report, alerts: report.alerts.map((a) => ({ ...a, ticket: tickets.get(a.id) ?? null })) });
// Who has said they are on it, and the incident record each alert is part of. Neither may stop the page from loading.
let acks = new Map<string, Awaited<ReturnType<typeof loadAcks>> extends Map<string, infer V> ? V : never>();
let incidents = new Map<string, number>();
try {
acks = await loadAcks();
incidents = await openIncidentIds(report.alerts.map((a) => a.id));
} catch (err) {
console.error("[alerts] couldn't look up acknowledgements:", err);
}
const attention = attentionCounts(report.alerts, acks);
res.json({
...report,
// `counts` still counts everything, as /metrics and the wall do; `attention` leaves out what somebody has said they are dealing with.
counts: { ...report.counts, acknowledged: attention.acknowledged, attention: { critical: attention.critical, warning: attention.warning } },
alerts: report.alerts.map((a) => ({ ...a, ticket: tickets.get(a.id) ?? null, ack: acks.get(a.id) ?? null, incidentId: incidents.get(a.id) ?? null })),
});
}));
// Who an alert can be handed to.
alertsRouter.get("/assignees", requireRole("operator"), asyncHandler(async (_req, res) => {
res.json({ users: await assignableUsers() });
}));
const ackSchema = z.object({
alertId: z.string().min(1).max(600),
note: z.string().max(MAX_NOTE).default(""),
assigneeId: z.number().int().positive().nullable().default(null),
/** How long it lasts; null (or left out) is until the alert is gone. */
hours: z.number().int().min(1).max(MAX_HOURS).nullable().default(null),
});
const auditTargetOf = (alert: { target: { type: string; id: number } | null }) => (alert.target ? { targetType: alert.target.type, targetId: alert.target.id } : {});
// "I'm on it": acknowledge an alert, say why, hand it to someone, and say how long it should be left alone.
alertsRouter.put("/ack", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = ackSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "Say which alert, and optionally why, who has it and for how long.", details: parsed.error.flatten() });
const alert = (await getAlerts(false)).alerts.find((a) => a.id === parsed.data.alertId);
if (!alert) return res.status(404).json({ error: "not_found", message: "That alert isn't there any more." });
const actor = req.currentUser!;
try {
const { ack, before } = await setAck({ ...parsed.data, actor: actor.name ?? actor.email ?? actor.oidcSub });
await recordAudit({
actor,
category: "alert",
action: "ack",
...auditTargetOf(alert),
detail: { alertId: alert.id, message: alert.message, note: ack.note, assignee: ack.assignee?.name ?? null, hours: parsed.data.hours, changed: before !== null },
});
res.json({ ack });
} catch (err) {
if (err instanceof AckError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
alertsRouter.delete("/ack", requireRole("operator"), asyncHandler(async (req, res) => {
const alertId = typeof req.query.alertId === "string" ? req.query.alertId : "";
if (!alertId) return res.status(400).json({ error: "invalid_id" });
const removed = await clearAck(alertId);
if (!removed) return res.status(404).json({ error: "not_found" });
const alert = (await getAlerts(false)).alerts.find((a) => a.id === alertId);
await recordAudit({ actor: req.currentUser!, category: "alert", action: "unack", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: alert?.message ?? null } });
res.status(204).end();
}));
+135
View File
@@ -0,0 +1,135 @@
import express, { Router } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
import {
backupDir,
cancelStagedRestore,
checkSnapshotFile,
createSnapshot,
deleteSnapshot,
dismissLastRestore,
importUploaded,
lastAttempt,
lastRestore,
listSnapshots,
pruneSnapshots,
snapshotPath,
stageRestore,
stagedRestore,
} from "../services/selfBackup.js";
import { nextBackupRun, scheduleSelfBackup } from "../services/selfBackupScheduler.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const dbBackupsRouter = Router();
// A backup holds everything the app knows — the audit log, every server's address, the hashed agent tokens and all the stored credentials
// (encrypted) — so all of it, looking and doing, is for administrators.
dbBackupsRouter.use(requireAuth, requireRole("admin"));
/** A database file is far bigger than anything else this app is sent, and is only ever read from an administrator. */
const UPLOAD_LIMIT = "1gb";
const settingsSchema = z.object({
enabled: z.boolean(),
time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/, "Use a time like 03:30"),
keep: z.number().int().min(1).max(90),
});
const actorName = (req: express.Request) => req.currentUser!.name ?? req.currentUser!.email ?? req.currentUser!.oidcSub;
async function overview() {
const { selfBackup } = await getSettings();
return {
settings: selfBackup,
snapshots: listSnapshots(),
staged: stagedRestore(),
lastRestore: lastRestore(),
lastAttempt: await lastAttempt(),
nextRun: nextBackupRun()?.toISOString() ?? null,
folder: backupDir(),
};
}
dbBackupsRouter.get("/", asyncHandler(async (_req, res) => {
res.json(await overview());
}));
dbBackupsRouter.put("/settings", asyncHandler(async (req, res) => {
const parsed = settingsSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "A time and how many to keep.", details: parsed.error.flatten() });
const before = (await getSettings()).selfBackup;
await updateSettings({ selfBackup: parsed.data });
await scheduleSelfBackup();
pruneSnapshots(parsed.data.keep);
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "settings", targetType: "settings", detail: { from: before, to: parsed.data } });
res.json(await overview());
}));
// Take one now. It is checked as it is made, and pruned like the scheduled ones.
dbBackupsRouter.post("/", asyncHandler(async (req, res) => {
try {
const { info, check } = await createSnapshot("manual");
pruneSnapshots((await getSettings()).selfBackup.keep);
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "create", targetType: "db_backup", targetId: info.name, detail: { sizeBytes: info.sizeBytes } });
res.status(201).json({ snapshot: info, check });
} catch (err) {
res.status(500).json({ error: "backup_failed", message: err instanceof Error ? err.message : String(err) });
}
}));
// A backup somebody has (from another machine, from the script in BACKUP.md): kept if it checks out, and can then be restored like any other.
dbBackupsRouter.post("/upload", express.raw({ type: () => true, limit: UPLOAD_LIMIT }), asyncHandler(async (req, res) => {
if (!Buffer.isBuffer(req.body) || req.body.length === 0) return res.status(400).json({ error: "no_file", message: "Choose a backup file to upload." });
try {
const { info, check } = await importUploaded(req.body);
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "upload", targetType: "db_backup", targetId: info.name, detail: { sizeBytes: info.sizeBytes } });
res.status(201).json({ snapshot: info, check });
} catch (err) {
res.status(400).json({ error: "unusable", message: err instanceof Error ? err.message : String(err) });
}
}));
dbBackupsRouter.post("/restore/cancel", asyncHandler(async (req, res) => {
const had = cancelStagedRestore();
if (had) await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "restore_cancelled" });
res.json({ cancelled: had });
}));
dbBackupsRouter.post("/restore-result/dismiss", asyncHandler(async (_req, res) => {
dismissLastRestore();
res.json({ ok: true });
}));
dbBackupsRouter.post("/:name/check", asyncHandler(async (req, res) => {
const file = snapshotPath(req.params.name);
if (!file) return res.status(404).json({ error: "not_found" });
res.json({ check: await checkSnapshotFile(file) });
}));
dbBackupsRouter.get("/:name/download", asyncHandler(async (req, res) => {
const file = snapshotPath(req.params.name);
if (!file) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "download", targetType: "db_backup", targetId: req.params.name });
res.setHeader("Cache-Control", "no-store");
res.download(file, req.params.name);
}));
// Checks the backup and stages it to replace the database the next time the app starts. Nothing changes until then, and the database in
// use is kept (as a "pre-restore" backup) when it does.
dbBackupsRouter.post("/:name/restore", asyncHandler(async (req, res) => {
try {
const check = await stageRestore(req.params.name, actorName(req));
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "restore_staged", targetType: "db_backup", targetId: req.params.name });
res.json({ staged: stagedRestore(), check });
} catch (err) {
res.status(400).json({ error: "cannot_restore", message: err instanceof Error ? err.message : String(err) });
}
}));
dbBackupsRouter.delete("/:name", asyncHandler(async (req, res) => {
if (!deleteSnapshot(req.params.name)) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "delete", targetType: "db_backup", targetId: req.params.name });
res.status(204).end();
}));
+51
View File
@@ -0,0 +1,51 @@
import { Router } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { addNote, getIncident, listIncidents, MAX_NOTE, removeNote } from "../services/incidents.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const incidentsRouter = Router();
// Everyone signed in can read what happened — it is made of what the Alerts page, the server pages and the Maintenance page already show.
// Writing a note is for operators.
incidentsRouter.use(requireAuth);
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
incidentsRouter.get("/", asyncHandler(async (_req, res) => {
res.json({ incidents: await listIncidents() });
}));
incidentsRouter.get("/:id", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const detail = await getIncident(id);
if (!detail) return res.status(404).json({ error: "not_found" });
res.json(detail);
}));
const noteSchema = z.object({ body: z.string().trim().min(1, "Write something first.").max(MAX_NOTE) });
incidentsRouter.post("/:id/notes", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = noteSchema.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.success ? undefined : parsed.error.issues[0]?.message });
const actor = req.currentUser!;
const note = await addNote(id, actor.name ?? actor.email ?? actor.oidcSub, parsed.data.body);
if (!note) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor, category: "incident", action: "note", targetType: "incident", targetId: id });
res.status(201).json({ id: note.id });
}));
incidentsRouter.delete("/:id/notes/:noteId", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const noteId = idOf(req.params.noteId);
if (!Number.isInteger(id) || !Number.isInteger(noteId)) return res.status(400).json({ error: "invalid_id" });
const actor = req.currentUser!;
const outcome = await removeNote(id, noteId, { name: actor.name ?? actor.email ?? actor.oidcSub, isAdmin: actor.role === "admin" });
if (outcome === "not_found") return res.status(404).json({ error: "not_found" });
if (outcome === "forbidden") return res.status(403).json({ error: "forbidden", message: "A note can be removed by the person who wrote it, or an administrator." });
await recordAudit({ actor, category: "incident", action: "note_removed", targetType: "incident", targetId: id });
res.status(204).end();
}));