Acknowledge and hand over alerts, incident timelines, and built-in database backups
- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics - Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents, incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs, related alerts and notes; Incidents page and a digest section - Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download, check, upload, delete, and restore staged and applied at start-up before the database opens (pre-restore copy kept); alerts and /metrics for stale or failing backups - Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7a5b623444
commit
2744a84719
41 files changed
+6573
-26
No files matched your search
@@ -1,6 +1,10 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { z } from "zod";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { getAlerts } from "../services/alerts.js";
|
||||
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { openIncidentIds } from "../services/incidents.js";
|
||||
import { ticketsForAlerts, type AlertTicketInfo } from "../services/alertTickets.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
@@ -19,5 +23,68 @@ alertsRouter.get("/", asyncHandler(async (req, res) => {
|
||||
} catch (err) {
|
||||
console.error("[alerts] couldn't look up tickets:", err);
|
||||
}
|
||||
res.json({ ...report, alerts: report.alerts.map((a) => ({ ...a, ticket: tickets.get(a.id) ?? null })) });
|
||||
// Who has said they are on it, and the incident record each alert is part of. Neither may stop the page from loading.
|
||||
let acks = new Map<string, Awaited<ReturnType<typeof loadAcks>> extends Map<string, infer V> ? V : never>();
|
||||
let incidents = new Map<string, number>();
|
||||
try {
|
||||
acks = await loadAcks();
|
||||
incidents = await openIncidentIds(report.alerts.map((a) => a.id));
|
||||
} catch (err) {
|
||||
console.error("[alerts] couldn't look up acknowledgements:", err);
|
||||
}
|
||||
const attention = attentionCounts(report.alerts, acks);
|
||||
res.json({
|
||||
...report,
|
||||
// `counts` still counts everything, as /metrics and the wall do; `attention` leaves out what somebody has said they are dealing with.
|
||||
counts: { ...report.counts, acknowledged: attention.acknowledged, attention: { critical: attention.critical, warning: attention.warning } },
|
||||
alerts: report.alerts.map((a) => ({ ...a, ticket: tickets.get(a.id) ?? null, ack: acks.get(a.id) ?? null, incidentId: incidents.get(a.id) ?? null })),
|
||||
});
|
||||
}));
|
||||
|
||||
// Who an alert can be handed to.
|
||||
alertsRouter.get("/assignees", requireRole("operator"), asyncHandler(async (_req, res) => {
|
||||
res.json({ users: await assignableUsers() });
|
||||
}));
|
||||
|
||||
const ackSchema = z.object({
|
||||
alertId: z.string().min(1).max(600),
|
||||
note: z.string().max(MAX_NOTE).default(""),
|
||||
assigneeId: z.number().int().positive().nullable().default(null),
|
||||
/** How long it lasts; null (or left out) is until the alert is gone. */
|
||||
hours: z.number().int().min(1).max(MAX_HOURS).nullable().default(null),
|
||||
});
|
||||
|
||||
const auditTargetOf = (alert: { target: { type: string; id: number } | null }) => (alert.target ? { targetType: alert.target.type, targetId: alert.target.id } : {});
|
||||
|
||||
// "I'm on it": acknowledge an alert, say why, hand it to someone, and say how long it should be left alone.
|
||||
alertsRouter.put("/ack", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = ackSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "Say which alert, and optionally why, who has it and for how long.", details: parsed.error.flatten() });
|
||||
const alert = (await getAlerts(false)).alerts.find((a) => a.id === parsed.data.alertId);
|
||||
if (!alert) return res.status(404).json({ error: "not_found", message: "That alert isn't there any more." });
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
const { ack, before } = await setAck({ ...parsed.data, actor: actor.name ?? actor.email ?? actor.oidcSub });
|
||||
await recordAudit({
|
||||
actor,
|
||||
category: "alert",
|
||||
action: "ack",
|
||||
...auditTargetOf(alert),
|
||||
detail: { alertId: alert.id, message: alert.message, note: ack.note, assignee: ack.assignee?.name ?? null, hours: parsed.data.hours, changed: before !== null },
|
||||
});
|
||||
res.json({ ack });
|
||||
} catch (err) {
|
||||
if (err instanceof AckError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
alertsRouter.delete("/ack", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const alertId = typeof req.query.alertId === "string" ? req.query.alertId : "";
|
||||
if (!alertId) return res.status(400).json({ error: "invalid_id" });
|
||||
const removed = await clearAck(alertId);
|
||||
if (!removed) return res.status(404).json({ error: "not_found" });
|
||||
const alert = (await getAlerts(false)).alerts.find((a) => a.id === alertId);
|
||||
await recordAudit({ actor: req.currentUser!, category: "alert", action: "unack", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: alert?.message ?? null } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -0,0 +1,135 @@
|
||||
import express, { Router } from "express";
|
||||
import { z } from "zod";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
import {
|
||||
backupDir,
|
||||
cancelStagedRestore,
|
||||
checkSnapshotFile,
|
||||
createSnapshot,
|
||||
deleteSnapshot,
|
||||
dismissLastRestore,
|
||||
importUploaded,
|
||||
lastAttempt,
|
||||
lastRestore,
|
||||
listSnapshots,
|
||||
pruneSnapshots,
|
||||
snapshotPath,
|
||||
stageRestore,
|
||||
stagedRestore,
|
||||
} from "../services/selfBackup.js";
|
||||
import { nextBackupRun, scheduleSelfBackup } from "../services/selfBackupScheduler.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const dbBackupsRouter = Router();
|
||||
|
||||
// A backup holds everything the app knows — the audit log, every server's address, the hashed agent tokens and all the stored credentials
|
||||
// (encrypted) — so all of it, looking and doing, is for administrators.
|
||||
dbBackupsRouter.use(requireAuth, requireRole("admin"));
|
||||
|
||||
/** A database file is far bigger than anything else this app is sent, and is only ever read from an administrator. */
|
||||
const UPLOAD_LIMIT = "1gb";
|
||||
|
||||
const settingsSchema = z.object({
|
||||
enabled: z.boolean(),
|
||||
time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/, "Use a time like 03:30"),
|
||||
keep: z.number().int().min(1).max(90),
|
||||
});
|
||||
|
||||
const actorName = (req: express.Request) => req.currentUser!.name ?? req.currentUser!.email ?? req.currentUser!.oidcSub;
|
||||
|
||||
async function overview() {
|
||||
const { selfBackup } = await getSettings();
|
||||
return {
|
||||
settings: selfBackup,
|
||||
snapshots: listSnapshots(),
|
||||
staged: stagedRestore(),
|
||||
lastRestore: lastRestore(),
|
||||
lastAttempt: await lastAttempt(),
|
||||
nextRun: nextBackupRun()?.toISOString() ?? null,
|
||||
folder: backupDir(),
|
||||
};
|
||||
}
|
||||
|
||||
dbBackupsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
res.json(await overview());
|
||||
}));
|
||||
|
||||
dbBackupsRouter.put("/settings", asyncHandler(async (req, res) => {
|
||||
const parsed = settingsSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "A time and how many to keep.", details: parsed.error.flatten() });
|
||||
const before = (await getSettings()).selfBackup;
|
||||
await updateSettings({ selfBackup: parsed.data });
|
||||
await scheduleSelfBackup();
|
||||
pruneSnapshots(parsed.data.keep);
|
||||
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "settings", targetType: "settings", detail: { from: before, to: parsed.data } });
|
||||
res.json(await overview());
|
||||
}));
|
||||
|
||||
// Take one now. It is checked as it is made, and pruned like the scheduled ones.
|
||||
dbBackupsRouter.post("/", asyncHandler(async (req, res) => {
|
||||
try {
|
||||
const { info, check } = await createSnapshot("manual");
|
||||
pruneSnapshots((await getSettings()).selfBackup.keep);
|
||||
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "create", targetType: "db_backup", targetId: info.name, detail: { sizeBytes: info.sizeBytes } });
|
||||
res.status(201).json({ snapshot: info, check });
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: "backup_failed", message: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
}));
|
||||
|
||||
// A backup somebody has (from another machine, from the script in BACKUP.md): kept if it checks out, and can then be restored like any other.
|
||||
dbBackupsRouter.post("/upload", express.raw({ type: () => true, limit: UPLOAD_LIMIT }), asyncHandler(async (req, res) => {
|
||||
if (!Buffer.isBuffer(req.body) || req.body.length === 0) return res.status(400).json({ error: "no_file", message: "Choose a backup file to upload." });
|
||||
try {
|
||||
const { info, check } = await importUploaded(req.body);
|
||||
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "upload", targetType: "db_backup", targetId: info.name, detail: { sizeBytes: info.sizeBytes } });
|
||||
res.status(201).json({ snapshot: info, check });
|
||||
} catch (err) {
|
||||
res.status(400).json({ error: "unusable", message: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
}));
|
||||
|
||||
dbBackupsRouter.post("/restore/cancel", asyncHandler(async (req, res) => {
|
||||
const had = cancelStagedRestore();
|
||||
if (had) await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "restore_cancelled" });
|
||||
res.json({ cancelled: had });
|
||||
}));
|
||||
|
||||
dbBackupsRouter.post("/restore-result/dismiss", asyncHandler(async (_req, res) => {
|
||||
dismissLastRestore();
|
||||
res.json({ ok: true });
|
||||
}));
|
||||
|
||||
dbBackupsRouter.post("/:name/check", asyncHandler(async (req, res) => {
|
||||
const file = snapshotPath(req.params.name);
|
||||
if (!file) return res.status(404).json({ error: "not_found" });
|
||||
res.json({ check: await checkSnapshotFile(file) });
|
||||
}));
|
||||
|
||||
dbBackupsRouter.get("/:name/download", asyncHandler(async (req, res) => {
|
||||
const file = snapshotPath(req.params.name);
|
||||
if (!file) return res.status(404).json({ error: "not_found" });
|
||||
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "download", targetType: "db_backup", targetId: req.params.name });
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
res.download(file, req.params.name);
|
||||
}));
|
||||
|
||||
// Checks the backup and stages it to replace the database the next time the app starts. Nothing changes until then, and the database in
|
||||
// use is kept (as a "pre-restore" backup) when it does.
|
||||
dbBackupsRouter.post("/:name/restore", asyncHandler(async (req, res) => {
|
||||
try {
|
||||
const check = await stageRestore(req.params.name, actorName(req));
|
||||
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "restore_staged", targetType: "db_backup", targetId: req.params.name });
|
||||
res.json({ staged: stagedRestore(), check });
|
||||
} catch (err) {
|
||||
res.status(400).json({ error: "cannot_restore", message: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
}));
|
||||
|
||||
dbBackupsRouter.delete("/:name", asyncHandler(async (req, res) => {
|
||||
if (!deleteSnapshot(req.params.name)) return res.status(404).json({ error: "not_found" });
|
||||
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "delete", targetType: "db_backup", targetId: req.params.name });
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -0,0 +1,51 @@
|
||||
import { Router } from "express";
|
||||
import { z } from "zod";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { addNote, getIncident, listIncidents, MAX_NOTE, removeNote } from "../services/incidents.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const incidentsRouter = Router();
|
||||
|
||||
// Everyone signed in can read what happened — it is made of what the Alerts page, the server pages and the Maintenance page already show.
|
||||
// Writing a note is for operators.
|
||||
incidentsRouter.use(requireAuth);
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
incidentsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
res.json({ incidents: await listIncidents() });
|
||||
}));
|
||||
|
||||
incidentsRouter.get("/:id", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const detail = await getIncident(id);
|
||||
if (!detail) return res.status(404).json({ error: "not_found" });
|
||||
res.json(detail);
|
||||
}));
|
||||
|
||||
const noteSchema = z.object({ body: z.string().trim().min(1, "Write something first.").max(MAX_NOTE) });
|
||||
|
||||
incidentsRouter.post("/:id/notes", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = noteSchema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.success ? undefined : parsed.error.issues[0]?.message });
|
||||
const actor = req.currentUser!;
|
||||
const note = await addNote(id, actor.name ?? actor.email ?? actor.oidcSub, parsed.data.body);
|
||||
if (!note) return res.status(404).json({ error: "not_found" });
|
||||
await recordAudit({ actor, category: "incident", action: "note", targetType: "incident", targetId: id });
|
||||
res.status(201).json({ id: note.id });
|
||||
}));
|
||||
|
||||
incidentsRouter.delete("/:id/notes/:noteId", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const noteId = idOf(req.params.noteId);
|
||||
if (!Number.isInteger(id) || !Number.isInteger(noteId)) return res.status(400).json({ error: "invalid_id" });
|
||||
const actor = req.currentUser!;
|
||||
const outcome = await removeNote(id, noteId, { name: actor.name ?? actor.email ?? actor.oidcSub, isAdmin: actor.role === "admin" });
|
||||
if (outcome === "not_found") return res.status(404).json({ error: "not_found" });
|
||||
if (outcome === "forbidden") return res.status(403).json({ error: "forbidden", message: "A note can be removed by the person who wrote it, or an administrator." });
|
||||
await recordAudit({ actor, category: "incident", action: "note_removed", targetType: "incident", targetId: id });
|
||||
res.status(204).end();
|
||||
}));
|
||||
Reference in new issue
Block a user