Acknowledge and hand over alerts, incident timelines, and built-in database backups

- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what
  needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics
- Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents,
  incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs,
  related alerts and notes; Incidents page and a digest section
- Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download,
  check, upload, delete, and restore staged and applied at start-up before the database opens
  (pre-restore copy kept); alerts and /metrics for stale or failing backups
- Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 02:21:08 +02:00
1 parent 7a5b623444
commit 2744a84719
41 files changed
+6573 -26

No files matched your search

+4
View File
@@ -12,6 +12,8 @@ import { uptimeRouter } from "./routes/uptime.js";
import { shareLinksRouter } from "./routes/shareLinks.js";
import { runbooksRouter } from "./routes/runbooks.js";
import { updateRunsRouter } from "./routes/updateRuns.js";
import { dbBackupsRouter } from "./routes/dbBackups.js";
import { incidentsRouter } from "./routes/incidents.js";
import { inventoryRouter } from "./routes/inventory.js";
import { backupsRouter } from "./routes/backups.js";
import { metricsRouter, metricsTokensRouter } from "./routes/metrics.js";
@@ -100,6 +102,8 @@ export function createApp() {
app.use("/api/share-links", shareLinksRouter);
app.use("/api/runbooks", runbooksRouter);
app.use("/api/update-runs", updateRunsRouter);
app.use("/api/db-backups", dbBackupsRouter);
app.use("/api/incidents", incidentsRouter);
app.use("/api/inventory", inventoryRouter);
app.use("/api/backups", backupsRouter);
app.use("/api/metrics-tokens", metricsTokensRouter);
+71
View File
@@ -0,0 +1,71 @@
/**
* Finishes a restore that was staged from Settings → Backup. It has to happen before the database is opened, which is why it is a module
* of its own that index.ts imports first: by the time anything else loads, the file is already the one that was chosen.
*
* Nothing is deleted. The database that was in use is moved into the backups folder (as "…-pre-restore.sqlite") before the chosen one is put
* in its place, so a restore that wasn't what was wanted can itself be undone by restoring that file. If anything goes wrong half way, the
* old file is put back and the staged one is left where it is.
*/
import "../loadEnv.js";
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { resolveDataPath } from "../paths.js";
import { PENDING_FILE, PENDING_NOTE, RESULT_NOTE, type RestoreResult } from "./restoreFiles.js";
const stamp = (d: Date) => d.toISOString().slice(0, 19).replace(/[:T]/g, "-");
/** Puts a staged database in place, if one is waiting. Returns what it did, or null if nothing was staged. */
export function applyPendingRestore(dbFile: string, now: Date = new Date()): RestoreResult | null {
const dataDir = dirname(dbFile);
const pending = join(dataDir, PENDING_FILE);
if (!existsSync(pending)) return null;
// Only a file is ever put in place of the database.
if (!statSync(pending).isFile()) {
console.error(`[restore] ${PENDING_FILE} isn't a file, so it was left alone and the database is as it was.`);
return null;
}
let note: { from?: string; by?: string | null } = {};
try {
note = JSON.parse(readFileSync(join(dataDir, PENDING_NOTE), "utf8"));
} catch {
// the note only says where it came from; the file is what matters
}
const backups = join(dataDir, "backups");
let keptAs: string | null = null;
let moved = false;
try {
mkdirSync(backups, { recursive: true });
if (existsSync(dbFile)) {
let name = `homelab-manager-${stamp(now)}-pre-restore.sqlite`;
for (let n = 2; existsSync(join(backups, name)); n++) name = `homelab-manager-${stamp(now)}-pre-restore-${n}.sqlite`;
renameSync(dbFile, join(backups, name));
keptAs = name;
moved = true;
}
// A journal left by the old file would be taken for the new one's.
for (const suffix of ["-wal", "-shm", "-journal"]) rmSync(dbFile + suffix, { force: true });
renameSync(pending, dbFile);
} catch (err) {
if (moved && keptAs && !existsSync(dbFile)) {
try {
renameSync(join(backups, keptAs), dbFile);
} catch {
// nothing more can be done; the old file is still in the backups folder under its new name
}
}
console.error("[restore] couldn't put the staged backup in place; the database is as it was:", err instanceof Error ? err.message : err);
return null;
}
const result: RestoreResult = { restoredAt: now.toISOString(), from: note.from ?? "an uploaded backup", keptAs, by: note.by ?? null };
writeFileSync(join(dataDir, RESULT_NOTE), JSON.stringify(result));
rmSync(join(dataDir, PENDING_NOTE), { force: true });
console.log(`[restore] restored the database from ${result.from}${keptAs ? `; the one that was in use is kept as backups/${keptAs}` : ""}.`);
return result;
}
// Run it now, on the database this process is about to open.
applyPendingRestore(resolveDataPath(process.env.DATABASE_PATH ?? "../data/homelab-manager.sqlite"));
+1 -1
View File
@@ -12,4 +12,4 @@ const client = createClient({ url: `file:${dbPath}` });
await client.execute("PRAGMA foreign_keys = ON;");
export const db = drizzle(client, { schema });
export { client };
export { client, dbPath };
+13
View File
@@ -0,0 +1,13 @@
// The names a staged restore uses in the data folder. Kept apart from applyRestore.ts, which does something when it is imported.
export const PENDING_FILE = "restore-pending.sqlite";
export const PENDING_NOTE = "restore-pending.json";
export const RESULT_NOTE = "restore-result.json";
export interface RestoreResult {
restoredAt: string;
/** The backup that is now the database. */
from: string;
/** Where the database that was in use went. */
keptAs: string | null;
by: string | null;
}
+57
View File
@@ -572,6 +572,63 @@ export const runbooks = sqliteTable(
(t) => [index("runbooks_target_idx").on(t.targetType, t.targetId)],
);
// ─── Acknowledgements and incidents ─────────────────────────────────────────
// "I'm on it" for one alert — see services/alertAcks.ts. Keyed by the alert's stable id (not a foreign key to anything); gone once the alert has been.
export const alertAcks = sqliteTable(
"alert_acks",
{
id: integer("id").primaryKey({ autoIncrement: true }),
alertKey: text("alert_key").notNull(),
note: text("note").notNull().default(""),
assigneeUserId: integer("assignee_user_id").references(() => users.id, { onDelete: "set null" }),
assigneeName: text("assignee_name"), // as it was when handed over, so it still reads if the user is removed
ackedBy: text("acked_by").notNull(),
ackedAt: text("acked_at").notNull(), // ISO
until: text("until"), // ISO; null = until the alert is gone
lastSeenAt: text("last_seen_at").notNull(), // ISO; the last time the alert was there
},
(t) => [uniqueIndex("alert_acks_key_idx").on(t.alertKey)],
);
// One time an alert was raised and later went away — see services/incidents.ts. When the same alert comes back it is a new row.
export const alertIncidents = sqliteTable(
"alert_incidents",
{
id: integer("id").primaryKey({ autoIncrement: true }),
alertKey: text("alert_key").notNull(),
severity: text("severity").notNull(), // as it is now (or was when it ended)
peakSeverity: text("peak_severity").notNull(),
category: text("category").notNull(),
source: text("source").notNull(),
message: text("message").notNull(), // the latest wording
link: text("link"),
targetType: text("target_type"), // server | integration | dns_provider, when the alert is about one of those
targetId: integer("target_id"),
targetName: text("target_name"),
causedBy: text("caused_by"), // the host whose outage this was folded under
openedAt: text("opened_at").notNull(), // ISO
resolvedAt: text("resolved_at"), // ISO; null while it is open
lastSeenAt: text("last_seen_at").notNull(), // ISO
missingSince: text("missing_since"), // ISO; set while it has been absent from the last look(s), before it is called over
},
(t) => [index("alert_incidents_key_idx").on(t.alertKey), index("alert_incidents_opened_idx").on(t.openedAt), index("alert_incidents_target_idx").on(t.targetType, t.targetId)],
);
export const incidentNotes = sqliteTable(
"incident_notes",
{
id: integer("id").primaryKey({ autoIncrement: true }),
incidentId: integer("incident_id")
.notNull()
.references(() => alertIncidents.id, { onDelete: "cascade" }),
author: text("author").notNull(),
body: text("body").notNull(),
createdAt: text("created_at").notNull(), // ISO
},
(t) => [index("incident_notes_incident_idx").on(t.incidentId)],
);
// ─── Update runs: apply a server's updates and see it through ──────────────
// A playbook run in Semaphore for one server, wrapped in a maintenance window and followed until the server reports again — see
+7
View File
@@ -1,4 +1,6 @@
import "./loadEnv.js";
// Before anything opens the database: puts a restore that was staged from Settings → Backup in place.
import "./db/applyRestore.js";
import { env, warnIfAuthNotConfigured, checkSessionSecret } from "./env.js";
import { createApp } from "./app.js";
import { runMigrations } from "./db/migrate.js";
@@ -17,6 +19,9 @@ import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceScheduler.js";
import { initMaintenanceScheduleScheduler } from "./services/maintenanceScheduleScheduler.js";
import { initUpdateRuns } from "./services/updateRuns.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
import { initIncidentRecorder } from "./services/incidents.js";
import { getAlerts } from "./services/alerts.js";
{
// Sessions are signed with this secret. Without one, the app would sign them with a value that's in the source code.
@@ -50,6 +55,8 @@ await initAlertTicketScheduler();
await initKumaMaintenanceScheduler();
await initMaintenanceScheduleScheduler();
initUpdateRuns();
await initSelfBackupScheduler();
initIncidentRecorder(() => getAlerts(false));
createApp().listen(env.port, () => {
console.log(`homelab-manager listening on port ${env.port}`);
+69 -2
View File
@@ -1,6 +1,10 @@
import { Router } from "express";
import { requireAuth } from "../auth/middleware.js";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { getAlerts } from "../services/alerts.js";
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
import { recordAudit } from "../services/audit.js";
import { openIncidentIds } from "../services/incidents.js";
import { ticketsForAlerts, type AlertTicketInfo } from "../services/alertTickets.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -19,5 +23,68 @@ alertsRouter.get("/", asyncHandler(async (req, res) => {
} catch (err) {
console.error("[alerts] couldn't look up tickets:", err);
}
res.json({ ...report, alerts: report.alerts.map((a) => ({ ...a, ticket: tickets.get(a.id) ?? null })) });
// Who has said they are on it, and the incident record each alert is part of. Neither may stop the page from loading.
let acks = new Map<string, Awaited<ReturnType<typeof loadAcks>> extends Map<string, infer V> ? V : never>();
let incidents = new Map<string, number>();
try {
acks = await loadAcks();
incidents = await openIncidentIds(report.alerts.map((a) => a.id));
} catch (err) {
console.error("[alerts] couldn't look up acknowledgements:", err);
}
const attention = attentionCounts(report.alerts, acks);
res.json({
...report,
// `counts` still counts everything, as /metrics and the wall do; `attention` leaves out what somebody has said they are dealing with.
counts: { ...report.counts, acknowledged: attention.acknowledged, attention: { critical: attention.critical, warning: attention.warning } },
alerts: report.alerts.map((a) => ({ ...a, ticket: tickets.get(a.id) ?? null, ack: acks.get(a.id) ?? null, incidentId: incidents.get(a.id) ?? null })),
});
}));
// Who an alert can be handed to.
alertsRouter.get("/assignees", requireRole("operator"), asyncHandler(async (_req, res) => {
res.json({ users: await assignableUsers() });
}));
const ackSchema = z.object({
alertId: z.string().min(1).max(600),
note: z.string().max(MAX_NOTE).default(""),
assigneeId: z.number().int().positive().nullable().default(null),
/** How long it lasts; null (or left out) is until the alert is gone. */
hours: z.number().int().min(1).max(MAX_HOURS).nullable().default(null),
});
const auditTargetOf = (alert: { target: { type: string; id: number } | null }) => (alert.target ? { targetType: alert.target.type, targetId: alert.target.id } : {});
// "I'm on it": acknowledge an alert, say why, hand it to someone, and say how long it should be left alone.
alertsRouter.put("/ack", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = ackSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "Say which alert, and optionally why, who has it and for how long.", details: parsed.error.flatten() });
const alert = (await getAlerts(false)).alerts.find((a) => a.id === parsed.data.alertId);
if (!alert) return res.status(404).json({ error: "not_found", message: "That alert isn't there any more." });
const actor = req.currentUser!;
try {
const { ack, before } = await setAck({ ...parsed.data, actor: actor.name ?? actor.email ?? actor.oidcSub });
await recordAudit({
actor,
category: "alert",
action: "ack",
...auditTargetOf(alert),
detail: { alertId: alert.id, message: alert.message, note: ack.note, assignee: ack.assignee?.name ?? null, hours: parsed.data.hours, changed: before !== null },
});
res.json({ ack });
} catch (err) {
if (err instanceof AckError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
alertsRouter.delete("/ack", requireRole("operator"), asyncHandler(async (req, res) => {
const alertId = typeof req.query.alertId === "string" ? req.query.alertId : "";
if (!alertId) return res.status(400).json({ error: "invalid_id" });
const removed = await clearAck(alertId);
if (!removed) return res.status(404).json({ error: "not_found" });
const alert = (await getAlerts(false)).alerts.find((a) => a.id === alertId);
await recordAudit({ actor: req.currentUser!, category: "alert", action: "unack", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: alert?.message ?? null } });
res.status(204).end();
}));
+135
View File
@@ -0,0 +1,135 @@
import express, { Router } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
import {
backupDir,
cancelStagedRestore,
checkSnapshotFile,
createSnapshot,
deleteSnapshot,
dismissLastRestore,
importUploaded,
lastAttempt,
lastRestore,
listSnapshots,
pruneSnapshots,
snapshotPath,
stageRestore,
stagedRestore,
} from "../services/selfBackup.js";
import { nextBackupRun, scheduleSelfBackup } from "../services/selfBackupScheduler.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const dbBackupsRouter = Router();
// A backup holds everything the app knows — the audit log, every server's address, the hashed agent tokens and all the stored credentials
// (encrypted) — so all of it, looking and doing, is for administrators.
dbBackupsRouter.use(requireAuth, requireRole("admin"));
/** A database file is far bigger than anything else this app is sent, and is only ever read from an administrator. */
const UPLOAD_LIMIT = "1gb";
const settingsSchema = z.object({
enabled: z.boolean(),
time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/, "Use a time like 03:30"),
keep: z.number().int().min(1).max(90),
});
const actorName = (req: express.Request) => req.currentUser!.name ?? req.currentUser!.email ?? req.currentUser!.oidcSub;
async function overview() {
const { selfBackup } = await getSettings();
return {
settings: selfBackup,
snapshots: listSnapshots(),
staged: stagedRestore(),
lastRestore: lastRestore(),
lastAttempt: await lastAttempt(),
nextRun: nextBackupRun()?.toISOString() ?? null,
folder: backupDir(),
};
}
dbBackupsRouter.get("/", asyncHandler(async (_req, res) => {
res.json(await overview());
}));
dbBackupsRouter.put("/settings", asyncHandler(async (req, res) => {
const parsed = settingsSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "A time and how many to keep.", details: parsed.error.flatten() });
const before = (await getSettings()).selfBackup;
await updateSettings({ selfBackup: parsed.data });
await scheduleSelfBackup();
pruneSnapshots(parsed.data.keep);
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "settings", targetType: "settings", detail: { from: before, to: parsed.data } });
res.json(await overview());
}));
// Take one now. It is checked as it is made, and pruned like the scheduled ones.
dbBackupsRouter.post("/", asyncHandler(async (req, res) => {
try {
const { info, check } = await createSnapshot("manual");
pruneSnapshots((await getSettings()).selfBackup.keep);
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "create", targetType: "db_backup", targetId: info.name, detail: { sizeBytes: info.sizeBytes } });
res.status(201).json({ snapshot: info, check });
} catch (err) {
res.status(500).json({ error: "backup_failed", message: err instanceof Error ? err.message : String(err) });
}
}));
// A backup somebody has (from another machine, from the script in BACKUP.md): kept if it checks out, and can then be restored like any other.
dbBackupsRouter.post("/upload", express.raw({ type: () => true, limit: UPLOAD_LIMIT }), asyncHandler(async (req, res) => {
if (!Buffer.isBuffer(req.body) || req.body.length === 0) return res.status(400).json({ error: "no_file", message: "Choose a backup file to upload." });
try {
const { info, check } = await importUploaded(req.body);
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "upload", targetType: "db_backup", targetId: info.name, detail: { sizeBytes: info.sizeBytes } });
res.status(201).json({ snapshot: info, check });
} catch (err) {
res.status(400).json({ error: "unusable", message: err instanceof Error ? err.message : String(err) });
}
}));
dbBackupsRouter.post("/restore/cancel", asyncHandler(async (req, res) => {
const had = cancelStagedRestore();
if (had) await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "restore_cancelled" });
res.json({ cancelled: had });
}));
dbBackupsRouter.post("/restore-result/dismiss", asyncHandler(async (_req, res) => {
dismissLastRestore();
res.json({ ok: true });
}));
dbBackupsRouter.post("/:name/check", asyncHandler(async (req, res) => {
const file = snapshotPath(req.params.name);
if (!file) return res.status(404).json({ error: "not_found" });
res.json({ check: await checkSnapshotFile(file) });
}));
dbBackupsRouter.get("/:name/download", asyncHandler(async (req, res) => {
const file = snapshotPath(req.params.name);
if (!file) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "download", targetType: "db_backup", targetId: req.params.name });
res.setHeader("Cache-Control", "no-store");
res.download(file, req.params.name);
}));
// Checks the backup and stages it to replace the database the next time the app starts. Nothing changes until then, and the database in
// use is kept (as a "pre-restore" backup) when it does.
dbBackupsRouter.post("/:name/restore", asyncHandler(async (req, res) => {
try {
const check = await stageRestore(req.params.name, actorName(req));
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "restore_staged", targetType: "db_backup", targetId: req.params.name });
res.json({ staged: stagedRestore(), check });
} catch (err) {
res.status(400).json({ error: "cannot_restore", message: err instanceof Error ? err.message : String(err) });
}
}));
dbBackupsRouter.delete("/:name", asyncHandler(async (req, res) => {
if (!deleteSnapshot(req.params.name)) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "delete", targetType: "db_backup", targetId: req.params.name });
res.status(204).end();
}));
+51
View File
@@ -0,0 +1,51 @@
import { Router } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { addNote, getIncident, listIncidents, MAX_NOTE, removeNote } from "../services/incidents.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const incidentsRouter = Router();
// Everyone signed in can read what happened — it is made of what the Alerts page, the server pages and the Maintenance page already show.
// Writing a note is for operators.
incidentsRouter.use(requireAuth);
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
incidentsRouter.get("/", asyncHandler(async (_req, res) => {
res.json({ incidents: await listIncidents() });
}));
incidentsRouter.get("/:id", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const detail = await getIncident(id);
if (!detail) return res.status(404).json({ error: "not_found" });
res.json(detail);
}));
const noteSchema = z.object({ body: z.string().trim().min(1, "Write something first.").max(MAX_NOTE) });
incidentsRouter.post("/:id/notes", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = noteSchema.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.success ? undefined : parsed.error.issues[0]?.message });
const actor = req.currentUser!;
const note = await addNote(id, actor.name ?? actor.email ?? actor.oidcSub, parsed.data.body);
if (!note) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor, category: "incident", action: "note", targetType: "incident", targetId: id });
res.status(201).json({ id: note.id });
}));
incidentsRouter.delete("/:id/notes/:noteId", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const noteId = idOf(req.params.noteId);
if (!Number.isInteger(id) || !Number.isInteger(noteId)) return res.status(400).json({ error: "invalid_id" });
const actor = req.currentUser!;
const outcome = await removeNote(id, noteId, { name: actor.name ?? actor.email ?? actor.oidcSub, isAdmin: actor.role === "admin" });
if (outcome === "not_found") return res.status(404).json({ error: "not_found" });
if (outcome === "forbidden") return res.status(403).json({ error: "forbidden", message: "A note can be removed by the person who wrote it, or an administrator." });
await recordAudit({ actor, category: "incident", action: "note_removed", targetType: "incident", targetId: id });
res.status(204).end();
}));
+133
View File
@@ -0,0 +1,133 @@
/**
* "I'm on it": an acknowledgement says that somebody has seen an alert, may say who is dealing with it and why, and may run out after a
* while. It changes nothing about the problem — the alert is still raised, still counted by /metrics and still on the wall display — but
* the people looking at the app can tell what nobody has looked at yet from what somebody has, and the sidebar and dashboard count only the
* former.
*
* An acknowledgement belongs to the alert's id (the same stable key the Alerts page keys rows on). It goes away on its own once the alert
* has been gone for a while, so the same problem coming back later is a new alert that nobody has looked at, not one that is "still being
* dealt with" from last month.
*/
import { and, eq, inArray, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { alertAcks, users } from "../db/schema.js";
import type { Alert } from "./alertTypes.js";
export interface AckInfo {
note: string;
assignee: { id: number | null; name: string } | null;
ackedBy: string;
ackedAt: string;
/** ISO — when it runs out; null means it lasts until the alert is gone. */
until: string | null;
/** Still in force: has no end, or the end hasn't come. */
active: boolean;
}
/** An alert that has been absent this long loses its acknowledgement (a blip of a few minutes doesn't). */
export const GONE_AFTER_MS = 30 * 60_000;
export const MAX_NOTE = 500;
export const MAX_HOURS = 24 * 30;
export class AckError extends Error {
constructor(
readonly status: number,
readonly code: string,
message: string,
) {
super(message);
}
}
const shape = (r: typeof alertAcks.$inferSelect, now: Date): AckInfo => ({
note: r.note,
assignee: r.assigneeName ? { id: r.assigneeUserId, name: r.assigneeName } : null,
ackedBy: r.ackedBy,
ackedAt: r.ackedAt,
until: r.until,
active: r.until === null || Date.parse(r.until) > now.getTime(),
});
export async function loadAcks(now: Date = new Date()): Promise<Map<string, AckInfo>> {
const rows = await db.select().from(alertAcks);
return new Map(rows.map((r) => [r.alertKey, shape(r, now)]));
}
export async function loadAck(alertKey: string, now: Date = new Date()): Promise<AckInfo | null> {
const [row] = await db.select().from(alertAcks).where(eq(alertAcks.alertKey, alertKey)).limit(1);
return row ? shape(row, now) : null;
}
export interface SetAckInput {
alertId: string;
note: string;
/** A user to hand it to; null for nobody in particular. */
assigneeId: number | null;
/** How long the acknowledgement lasts; null means until the alert is gone. */
hours: number | null;
actor: string;
}
/** Who an alert can be handed to: people who can act on it. */
export async function assignableUsers(): Promise<{ id: number; name: string; role: string }[]> {
const rows = await db.select().from(users).where(inArray(users.role, ["admin", "operator"]));
return rows.map((u) => ({ id: u.id, name: u.name ?? u.email ?? u.oidcSub, role: u.role })).sort((a, b) => a.name.localeCompare(b.name));
}
/** Saves an acknowledgement, replacing an earlier one on the same alert. Returns what changed, for the audit log. */
export async function setAck(input: SetAckInput, now: Date = new Date()): Promise<{ ack: AckInfo; before: AckInfo | null }> {
const note = input.note.trim();
if (note.length > MAX_NOTE) throw new AckError(400, "note_too_long", `Keep the note under ${MAX_NOTE} characters.`);
if (input.hours !== null && !(Number.isInteger(input.hours) && input.hours >= 1 && input.hours <= MAX_HOURS)) throw new AckError(400, "invalid_duration", "Choose how long it lasts, up to thirty days — or until the alert is gone.");
let assigneeName: string | null = null;
if (input.assigneeId !== null) {
const [u] = await db.select().from(users).where(eq(users.id, input.assigneeId)).limit(1);
if (!u || (u.role !== "admin" && u.role !== "operator")) throw new AckError(400, "invalid_assignee", "An alert can be handed to an administrator or an operator.");
assigneeName = u.name ?? u.email ?? u.oidcSub;
}
const before = await loadAck(input.alertId, now);
const until = input.hours === null ? null : new Date(now.getTime() + input.hours * 3600_000).toISOString();
const values = { note, assigneeUserId: input.assigneeId, assigneeName, ackedBy: input.actor, ackedAt: now.toISOString(), until, lastSeenAt: now.toISOString() };
await db.insert(alertAcks).values({ alertKey: input.alertId, ...values }).onConflictDoUpdate({ target: alertAcks.alertKey, set: values });
return { ack: (await loadAck(input.alertId, now))!, before };
}
export async function clearAck(alertId: string): Promise<AckInfo | null> {
const before = await loadAck(alertId);
if (!before) return null;
await db.delete(alertAcks).where(eq(alertAcks.alertKey, alertId));
return before;
}
/**
* Called after every collection of the alerts: the acknowledgements of alerts that are there are marked as seen, and those of alerts that
* have been gone for half an hour are dropped. Returns how many were dropped.
*/
export async function reconcileAcks(presentIds: Set<string>, now: Date = new Date()): Promise<number> {
const rows = await db.select().from(alertAcks);
const seen = rows.filter((r) => presentIds.has(r.alertKey)).map((r) => r.id);
if (seen.length > 0) await db.update(alertAcks).set({ lastSeenAt: now.toISOString() }).where(inArray(alertAcks.id, seen));
const cutoff = new Date(now.getTime() - GONE_AFTER_MS).toISOString();
const gone = rows.filter((r) => !presentIds.has(r.alertKey) && r.lastSeenAt < cutoff).map((r) => r.id);
if (gone.length > 0) await db.delete(alertAcks).where(and(inArray(alertAcks.id, gone), lt(alertAcks.lastSeenAt, cutoff)));
return gone.length;
}
/** What still needs somebody to look: the alerts that aren't silenced and that nobody has acknowledged (or whose acknowledgement has run out). */
export function attentionCounts(alerts: Pick<Alert, "id" | "severity" | "silenced">[], acks: Map<string, AckInfo>): { critical: number; warning: number; acknowledged: number } {
let critical = 0;
let warning = 0;
let acknowledged = 0;
for (const a of alerts) {
if (a.silenced) continue;
if (acks.get(a.id)?.active) {
acknowledged++;
continue;
}
if (a.severity === "critical") critical++;
else if (a.severity === "warning") warning++;
}
return { critical, warning, acknowledged };
}
+29
View File
@@ -15,6 +15,9 @@ import { integrations, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
import { recordFromReport } from "./incidents.js";
import { reconcileAcks } from "./alertAcks.js";
import { lastAttempt as lastSelfBackup, newestSnapshot, STALE_AFTER_HOURS as SELF_BACKUP_STALE_HOURS } from "./selfBackup.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
@@ -53,6 +56,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
gitea: { label: "Gitea", link: "/gitea" },
dockhand: { label: "Docker", link: "/docker" },
tailscale: { label: "Tailscale", link: "/tailscale" },
homelab: { label: "Homelab Manager", link: "/settings/backup" },
pbs: { label: "Proxmox Backup", link: "/pbs" },
uptimekuma: { label: "Uptime Kuma", link: "/uptime-kuma" },
osticket: { label: "osTicket", link: "/osticket" },
@@ -262,6 +266,29 @@ export async function collectAlerts(): Promise<AlertsReport> {
}
}),
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
// stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it.
check("Homelab Manager's own backups", async () => {
const { selfBackup } = await getSettings();
if (!selfBackup.enabled) return;
const newest = newestSnapshot();
const last = await lastSelfBackup();
const failing = last && !last.ok ? ` The last attempt failed: ${last.error}` : "";
if (!newest) {
if (failing) push({ category: "backup", severity: "warning", type: "homelab", message: `Homelab Manager has no backup of its own yet.${failing}`, key: "self-backup" });
return;
}
const ageHours = (Date.now() - Date.parse(newest.at)) / 3600_000;
if (ageHours <= SELF_BACKUP_STALE_HOURS) return;
push({
category: "backup",
severity: ageHours >= 24 * 7 ? "critical" : "warning",
type: "homelab",
message: `The newest backup of Homelab Manager's own database is ${ageHours < 72 ? `${Math.round(ageHours)} h` : `${Math.floor(ageHours / 24)} days`} old.${failing}`,
key: "self-backup",
});
}),
check("Backup verification", async () => {
const { problems, sourceFailures } = await collectPbsProblems({ skipSilenced: false });
for (const p of problems) {
@@ -475,6 +502,8 @@ export async function getAlerts(force: boolean): Promise<AlertsReport & { cached
inFlight ??= collectAlerts()
.then((report) => {
cache = { at: Date.now(), report };
// Remembered for the incident record and the acknowledgements — in the background, and never allowed to hold up or break the answer.
void recordFromReport(report).then(() => reconcileAcks(new Set(report.alerts.map((a) => a.id)))).catch((err) => console.error("[alerts] couldn't update the incident record:", err));
return report;
})
.finally(() => {
+326
View File
@@ -0,0 +1,326 @@
/**
* Incidents: each time an alert is raised and later goes away is one incident, with the moment it began and the moment it ended — so
* "what happened on Tuesday night?" has an answer after the alert itself is long gone. Around each one the page puts together everything the
* app knows about that time: who acknowledged it and why, the maintenance windows, what was changed on the server, the update runs, the
* other alerts that came with it, and the notes people wrote.
*
* The record is made from the alerts as they are collected (and a check every five minutes so it doesn't depend on anybody looking), by
* comparing what is there now with what was open. An alert that isn't there is only called resolved after it has been missing for a couple
* of minutes in a row — much longer when a check couldn't run, since "not there" then may only mean "couldn't look" — and never in the first
* minutes after the app starts, when server alerts are held back on purpose.
*/
import { and, asc, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or } from "drizzle-orm";
import { db } from "../db/client.js";
import { alertIncidents, auditLog, incidentNotes, maintenanceWindows, updateRuns } from "../db/schema.js";
import type { Alert } from "./alertTypes.js";
import { describeAudit, auditTimeToIso, type ActivityTone } from "./serverActivity.js";
import { startupGraceRemainingMs } from "./healthMonitor.js";
import { loadAck, type AckInfo } from "./alertAcks.js";
export type IncidentRow = typeof alertIncidents.$inferSelect;
/** How long an alert must be missing before the incident is called over. */
export const MISSING_GRACE_MS = 2 * 60_000;
/** The same, when a check couldn't run on the last look: not there may then only mean couldn't look. */
export const MISSING_GRACE_BLIND_MS = 30 * 60_000;
/** Resolved incidents are kept this long. */
export const KEEP_DAYS = 400;
/** Other alerts raised this close to an incident, about the same thing, are shown with it. */
export const RELATED_WINDOW_MS = 15 * 60_000;
const RANK: Record<string, number> = { info: 0, warning: 1, critical: 2 };
export interface SnapshotInput {
alerts: Alert[];
/** The checks that couldn't run on this look. */
blind: number;
generatedAt?: string;
}
export interface SnapshotResult {
opened: number[];
resolved: number[];
}
/**
* Brings the incidents in line with the alerts as they are now. Information-only alerts aren't incidents. Safe to call as often as likes:
* the same look twice changes nothing.
*/
export async function recordAlertSnapshot(input: SnapshotInput, now: Date = new Date(), options: { inGrace?: boolean } = {}): Promise<SnapshotResult> {
const iso = now.toISOString();
const present = new Map(input.alerts.filter((a) => a.severity !== "info").map((a) => [a.id, a]));
const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt));
const openByKey = new Map(open.map((i) => [i.alertKey, i]));
const result: SnapshotResult = { opened: [], resolved: [] };
for (const [key, a] of present) {
const existing = openByKey.get(key);
if (existing) {
const peak = (RANK[a.severity] ?? 0) > (RANK[existing.peakSeverity] ?? 0) ? a.severity : existing.peakSeverity;
await db
.update(alertIncidents)
.set({ severity: a.severity, peakSeverity: peak, message: a.message, causedBy: a.causedBy?.name ?? null, lastSeenAt: iso, missingSince: null })
.where(eq(alertIncidents.id, existing.id));
} else {
const [row] = await db
.insert(alertIncidents)
.values({
alertKey: key,
severity: a.severity,
peakSeverity: a.severity,
category: a.category,
source: a.source,
message: a.message,
link: a.link,
targetType: a.target?.type ?? null,
targetId: a.target?.id ?? null,
targetName: a.target?.name ?? null,
causedBy: a.causedBy?.name ?? null,
openedAt: iso,
lastSeenAt: iso,
})
.returning({ id: alertIncidents.id });
result.opened.push(row.id);
}
}
// What was open and isn't there now. Nothing is called over while the app is still settling after a start, when server alerts are held back.
const grace = input.blind > 0 ? MISSING_GRACE_BLIND_MS : MISSING_GRACE_MS;
for (const inc of open) {
if (present.has(inc.alertKey)) continue;
if (options.inGrace) continue;
if (inc.missingSince === null) {
await db.update(alertIncidents).set({ missingSince: iso }).where(eq(alertIncidents.id, inc.id));
} else if (now.getTime() - Date.parse(inc.missingSince) >= grace) {
// Over since the first look it was missing, not since this one.
await db.update(alertIncidents).set({ resolvedAt: inc.missingSince, missingSince: null }).where(eq(alertIncidents.id, inc.id));
result.resolved.push(inc.id);
}
}
return result;
}
/** Called by the collection of the alerts: records what it found, and never lets a problem here stop the alerts from being served. */
export async function recordFromReport(report: { alerts: Alert[]; couldntCheck: unknown[] }, now: Date = new Date()): Promise<void> {
try {
await recordAlertSnapshot({ alerts: report.alerts, blind: report.couldntCheck.length }, now, { inGrace: startupGraceRemainingMs(now.getTime()) > 0 });
} catch (err) {
console.error("[incidents] couldn't record the alerts:", err);
}
}
/** Resolved incidents older than the keeping time are removed (with their notes). */
export async function pruneIncidents(now: Date = new Date()): Promise<number> {
const cutoff = new Date(now.getTime() - KEEP_DAYS * 86_400_000).toISOString();
const old = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(and(isNotNull(alertIncidents.resolvedAt), lte(alertIncidents.resolvedAt, cutoff)));
if (old.length === 0) return 0;
await db.delete(alertIncidents).where(inArray(alertIncidents.id, old.map((o) => o.id)));
return old.length;
}
// ─── Reading them ───────────────────────────────────────────────────────────
export interface IncidentSummary {
id: number;
alertKey: string;
severity: string;
peakSeverity: string;
category: string;
source: string;
message: string;
link: string | null;
target: { type: string; id: number; name: string } | null;
causedBy: string | null;
openedAt: string;
resolvedAt: string | null;
/** In milliseconds: until it was resolved, or until now while it is open. */
durationMs: number;
open: boolean;
notes: number;
}
const summarize = (r: IncidentRow, notes: number, now: Date): IncidentSummary => ({
id: r.id,
alertKey: r.alertKey,
severity: r.severity,
peakSeverity: r.peakSeverity,
category: r.category,
source: r.source,
message: r.message,
link: r.link,
target: r.targetType && r.targetId !== null ? { type: r.targetType, id: r.targetId, name: r.targetName ?? `${r.targetType} #${r.targetId}` } : null,
causedBy: r.causedBy,
openedAt: r.openedAt,
resolvedAt: r.resolvedAt,
durationMs: Math.max(0, (r.resolvedAt ? Date.parse(r.resolvedAt) : now.getTime()) - Date.parse(r.openedAt)),
open: r.resolvedAt === null,
notes,
});
/** Open incidents first (newest first), then the resolved ones, newest first. */
export async function listIncidents(options: { limit?: number; now?: Date } = {}): Promise<IncidentSummary[]> {
const now = options.now ?? new Date();
const limit = options.limit ?? 300;
const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt));
const closed = await db.select().from(alertIncidents).where(isNotNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt)).limit(limit);
const rows = [...open, ...closed];
const noteRows = rows.length > 0 ? await db.select({ incidentId: incidentNotes.incidentId }).from(incidentNotes).where(inArray(incidentNotes.incidentId, rows.map((r) => r.id))) : [];
const counts = new Map<number, number>();
for (const n of noteRows) counts.set(n.incidentId, (counts.get(n.incidentId) ?? 0) + 1);
return rows.map((r) => summarize(r, counts.get(r.id) ?? 0, now));
}
/** The open incident for each alert id, so the Alerts page can link to it. */
export async function openIncidentIds(alertKeys: string[]): Promise<Map<string, number>> {
if (alertKeys.length === 0) return new Map();
const rows = await db.select({ id: alertIncidents.id, key: alertIncidents.alertKey }).from(alertIncidents).where(and(isNull(alertIncidents.resolvedAt), inArray(alertIncidents.alertKey, alertKeys)));
return new Map(rows.map((r) => [r.key, r.id]));
}
export type TimelineKind = "alert" | "ack" | "maintenance" | "change" | "health" | "run" | "related" | "note";
export interface TimelineEvent {
/** ISO. */
at: string;
kind: TimelineKind;
tone: ActivityTone;
title: string;
detail: string | null;
actor: string | null;
/** For a related alert or a note: what it points at. */
incidentId?: number;
noteId?: number;
}
export interface IncidentDetail {
incident: IncidentSummary;
/** The acknowledgement in force now, if the alert is still open. */
ack: AckInfo | null;
timeline: TimelineEvent[];
notes: { id: number; author: string; body: string; at: string }[];
}
function parse(detail: string | null): Record<string, unknown> {
if (!detail) return {};
try {
const v = JSON.parse(detail);
return v && typeof v === "object" ? (v as Record<string, unknown>) : {};
} catch {
return {};
}
}
const words = (ms: number) => {
const m = Math.round(ms / 60_000);
if (m < 90) return `${Math.max(1, m)} min`;
const h = ms / 3600_000;
return h < 48 ? `${Math.round(h * 10) / 10} h` : `${Math.round(h / 24)} days`;
};
/** Everything the app knows about one incident, in the order it happened. */
export async function getIncident(id: number, now: Date = new Date()): Promise<IncidentDetail | null> {
const [row] = await db.select().from(alertIncidents).where(eq(alertIncidents.id, id)).limit(1);
if (!row) return null;
const notes = await db.select().from(incidentNotes).where(eq(incidentNotes.incidentId, id)).orderBy(asc(incidentNotes.createdAt));
const incident = summarize(row, notes.length, now);
const from = Date.parse(row.openedAt);
const to = row.resolvedAt ? Date.parse(row.resolvedAt) : now.getTime();
const fromIso = new Date(from - 30 * 60_000).toISOString();
const toIso = new Date(to + 15 * 60_000).toISOString();
const events: TimelineEvent[] = [];
events.push({ at: row.openedAt, kind: "alert", tone: row.peakSeverity === "critical" ? "bad" : "info", title: `Alert raised (${row.severity})`, detail: row.message, actor: null });
if (row.peakSeverity !== row.severity) {
events.push({ at: row.resolvedAt ?? row.lastSeenAt, kind: "alert", tone: "neutral", title: `At its worst it was ${row.peakSeverity}`, detail: null, actor: null });
}
if (row.causedBy) events.push({ at: row.openedAt, kind: "alert", tone: "info", title: `Held back: caused by ${row.causedBy}`, detail: "Folded under the alert for the machine it runs on.", actor: null });
if (row.resolvedAt) events.push({ at: row.resolvedAt, kind: "alert", tone: "good", title: "Resolved", detail: `It lasted ${words(to - from)}.`, actor: null });
// Acknowledgements and hand-overs of this alert, from the audit log.
const audits = await db.select().from(auditLog).where(and(gte(auditLog.createdAt, fromIso.slice(0, 19).replace("T", " ")), lte(auditLog.createdAt, toIso.slice(0, 19).replace("T", " ")))).orderBy(asc(auditLog.createdAt));
for (const a of audits) {
const d = parse(a.detail);
if (a.category === "alert") {
if (d.alertId !== row.alertKey) continue;
const described = describeAudit(a.category, a.action, d);
events.push({ at: auditTimeToIso(a.createdAt), kind: "ack", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel });
continue;
}
// Things done to, or noticed about, the thing the alert is about.
if (row.targetType && row.targetId !== null && a.targetType === row.targetType && a.targetId === String(row.targetId) && a.category !== "maintenance") {
const described = describeAudit(a.category, a.action, d);
events.push({ at: auditTimeToIso(a.createdAt), kind: described.kind === "health" ? "health" : "change", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel });
}
}
if (row.targetType && row.targetId !== null) {
const windows = await db
.select()
.from(maintenanceWindows)
.where(and(eq(maintenanceWindows.targetType, row.targetType as "server"), eq(maintenanceWindows.targetId, row.targetId), lte(maintenanceWindows.startedAt, new Date(to).toISOString()), gte(maintenanceWindows.endsAt, new Date(from).toISOString())));
for (const w of windows) {
if (w.endsAt <= w.startedAt) continue; // withdrawn before it began
events.push({ at: w.startedAt, kind: "maintenance", tone: "info", title: "Maintenance began — alerts silenced", detail: w.reason, actor: w.createdBy });
if (w.endsAt <= toIso) events.push({ at: w.endsAt, kind: "maintenance", tone: "neutral", title: "Maintenance ended", detail: null, actor: null });
}
if (row.targetType === "server") {
const runs = await db.select().from(updateRuns).where(and(eq(updateRuns.serverId, row.targetId), lte(updateRuns.startedAt, new Date(to).toISOString()), or(isNull(updateRuns.finishedAt), gte(updateRuns.finishedAt, new Date(from).toISOString()))!));
for (const r of runs) {
events.push({ at: r.startedAt, kind: "run", tone: "info", title: `Update run started: ${r.templateName}`, detail: `by ${r.startedBy ?? "someone"}`, actor: r.startedBy });
if (r.finishedAt) events.push({ at: r.finishedAt, kind: "run", tone: r.status === "done" ? "good" : "bad", title: r.status === "done" ? "Update run finished" : r.status === "cancelled" ? "Update run stopped being followed" : "Update run didn't finish", detail: r.message, actor: null });
}
}
// Other alerts about the same thing that began around the same time.
const related = await db
.select()
.from(alertIncidents)
.where(and(eq(alertIncidents.targetType, row.targetType), eq(alertIncidents.targetId, row.targetId), ne(alertIncidents.id, row.id), gte(alertIncidents.openedAt, new Date(from - RELATED_WINDOW_MS).toISOString()), lte(alertIncidents.openedAt, new Date(Math.max(to, from) + RELATED_WINDOW_MS).toISOString())));
for (const r of related) events.push({ at: r.openedAt, kind: "related", tone: "info", title: `Another alert was raised: ${r.category} (${r.severity})`, detail: r.message, actor: null, incidentId: r.id });
}
for (const n of notes) events.push({ at: n.createdAt, kind: "note", tone: "neutral", title: `${n.author} wrote a note`, detail: n.body, actor: n.author, noteId: n.id });
events.sort((a, b) => a.at.localeCompare(b.at));
return { incident, ack: row.resolvedAt ? null : await loadAck(row.alertKey, now), timeline: events, notes: notes.map((n) => ({ id: n.id, author: n.author, body: n.body, at: n.createdAt })) };
}
export const MAX_NOTE = 4000;
export async function addNote(incidentId: number, author: string, body: string, now: Date = new Date()): Promise<{ id: number } | null> {
const [inc] = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(eq(alertIncidents.id, incidentId)).limit(1);
if (!inc) return null;
const [row] = await db.insert(incidentNotes).values({ incidentId, author, body: body.trim(), createdAt: now.toISOString() }).returning({ id: incidentNotes.id });
return row;
}
export async function removeNote(incidentId: number, noteId: number, actor: { name: string; isAdmin: boolean }): Promise<"removed" | "not_found" | "forbidden"> {
const [n] = await db.select().from(incidentNotes).where(and(eq(incidentNotes.id, noteId), eq(incidentNotes.incidentId, incidentId))).limit(1);
if (!n) return "not_found";
if (n.author !== actor.name && !actor.isAdmin) return "forbidden";
await db.delete(incidentNotes).where(eq(incidentNotes.id, noteId));
return "removed";
}
let timer: ReturnType<typeof setInterval> | null = null;
/** A look at the alerts every five minutes, so incidents are recorded whether or not anybody has the page open. Once a day it tidies up old ones. */
export function initIncidentRecorder(lookAtAlerts: () => Promise<unknown>): void {
if (timer) return;
let lastPrune = 0;
const tick = async () => {
try {
await lookAtAlerts();
if (Date.now() - lastPrune > 24 * 3600_000) {
lastPrune = Date.now();
await pruneIncidents();
}
} catch (err) {
console.error("[incidents] the five-minute look failed:", err);
}
};
setTimeout(() => void tick(), 60_000).unref();
timer = setInterval(() => void tick(), 5 * 60_000);
timer.unref();
}
+18
View File
@@ -13,6 +13,9 @@ import { parsePendingUpdates } from "./pendingUpdates.js";
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
import { parseThresholds } from "./serverThresholds.js";
import { getCachedBackupCoverage } from "./backupCoverage.js";
import { lastAttempt as lastSelfBackup, listSnapshots } from "./selfBackup.js";
import { loadAcks, attentionCounts } from "./alertAcks.js";
import { listIncidents } from "./incidents.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { computeSecretStatus } from "./secretStatus.js";
import { getSettings } from "./settingsStore.js";
@@ -115,6 +118,18 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
console.error("[metrics] couldn't read the uptime:", err);
}
// ── Homelab Manager's own backups ──
try {
const snapshots = listSnapshots();
out.add("homelab_selfbackup_snapshots", "gauge", "How many backups of Homelab Manager's own database are kept.", snapshots.length);
const newest = snapshots.find((s) => s.kind === "scheduled" || s.kind === "manual");
if (newest) out.add("homelab_selfbackup_last_success_timestamp_seconds", "gauge", "When the newest backup of Homelab Manager's own database was made, as a Unix time.", Math.floor(Date.parse(newest.at) / 1000));
const last = await lastSelfBackup();
if (last) out.add("homelab_selfbackup_last_attempt_ok", "gauge", "1 if the last attempt to back up Homelab Manager's own database worked, 0 if it failed.", last.ok ? 1 : 0);
} catch (err) {
console.error("[metrics] couldn't read the backups folder:", err);
}
// ── Alerts ──
try {
const report = await getAlerts(false);
@@ -127,6 +142,9 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
}
}
for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity });
// Somebody has said they are dealing with these (still raised, and still counted above).
out.add("homelab_alerts_acknowledged", "gauge", "Alerts somebody has acknowledged and that are not silenced by maintenance.", attentionCounts(report.alerts, await loadAcks(now)).acknowledged);
out.add("homelab_incidents_open", "gauge", "Incidents that have begun and not yet ended.", (await listIncidents({ limit: 1, now })).filter((i) => i.open).length);
out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced);
out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped);
} catch (err) {
+323
View File
@@ -0,0 +1,323 @@
/**
* Backups of Homelab Manager itself. Everything it knows is one SQLite file, so a backup is a consistent copy of that file (`VACUUM INTO`,
* which is safe while the app is working) kept in the data folder's `backups/`: taken every day, the newest few kept, each one checked as
* it is made, and downloadable, uploadable and restorable from Settings → Backup.
*
* What a backup is not: it does not hold `CREDENTIALS_ENCRYPTION_KEY`, which decrypts the credentials stored in it, and it isn't off the
* machine until someone copies it there. Both are said where backups are shown.
*
* A restore can't swap a database that is open, so it is staged (checked, then copied beside the database) and takes effect when the app
* next starts — see db/applyRestore.ts, which does it before anything opens the file.
*/
import { createClient } from "@libsql/client";
import { copyFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, openSync, readdirSync, readFileSync, readSync, closeSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { client, dbPath } from "../db/client.js";
import { PENDING_FILE, PENDING_NOTE, RESULT_NOTE, type RestoreResult } from "../db/restoreFiles.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export type SnapshotKind = "scheduled" | "manual" | "pre-restore" | "uploaded";
export interface SnapshotInfo {
name: string;
kind: SnapshotKind;
sizeBytes: number;
/** ISO — when the file was written. */
at: string;
}
export interface SnapshotCheck {
ok: boolean;
/** SQLite's own verdict ("ok", or what it found). */
integrity: string | null;
/** How many schema changes it has had; goes up with each release that changes the database. */
migrations: number | null;
/** How many this version of the app knows. */
appMigrations: number;
/** Made by a newer version of the app than this one: it can't be restored here. */
newerThanApp: boolean;
counts: { users: number; servers: number; integrations: number } | null;
/** Why it isn't usable, when it isn't. */
problem: string | null;
}
const NAME = /^homelab-manager-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}(?:-(manual|pre-restore|uploaded))?(?:-\d+)?\.sqlite$/;
export const isSnapshotName = (name: string) => NAME.test(name);
const dataDir = () => dirname(dbPath);
export const backupDir = () => join(dataDir(), "backups");
const stamp = (d: Date) => d.toISOString().slice(0, 19).replace(/[:T]/g, "-");
function kindOf(name: string): SnapshotKind {
const m = NAME.exec(name);
return (m?.[1] as SnapshotKind | undefined) ?? "scheduled";
}
/** How many schema changes this version of the app has (the entries in its migration journal). */
export function appMigrationCount(): number {
try {
const journal = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "drizzle", "meta", "_journal.json"), "utf8")) as { entries?: unknown[] };
return journal.entries?.length ?? 0;
} catch {
return 0;
}
}
function hasSqliteHeader(file: string): boolean {
try {
const fd = openSync(file, "r");
try {
const buf = Buffer.alloc(16);
readSync(fd, buf, 0, 16, 0);
return buf.toString("latin1") === "SQLite format 3\0";
} finally {
closeSync(fd);
}
} catch {
return false;
}
}
/** A scratch folder for looking at a copy of a backup, and the way to be rid of it (best effort: some systems hold an opened database until the process ends). */
function scratch(): { file: string; done: () => void } {
const folder = mkdtempSync(join(tmpdir(), "hlm-backup-"));
return {
file: join(folder, "db.sqlite"),
done: () => {
try {
rmSync(folder, { recursive: true, force: true });
} catch {
// left for the system's own tidying of its temporary folder
}
},
};
}
/**
* Looks at a backup: does SQLite think it is sound, is it one of ours, and is it from this version or an older one? It looks at a copy in a
* temporary folder, never at the file itself, so a backup in the backups folder is never held open — it can always be deleted, anywhere.
*/
export async function checkSnapshotFile(file: string): Promise<SnapshotCheck> {
if (!existsSync(file)) return checkOpened(file);
if (!hasSqliteHeader(file)) return checkOpened(file);
const tmp = scratch();
try {
copyFileSync(file, tmp.file);
return await checkOpened(tmp.file);
} finally {
tmp.done();
}
}
/** The check itself, on a file nobody else cares about the handle of. */
async function checkOpened(file: string): Promise<SnapshotCheck> {
const appMigrations = appMigrationCount();
const fail = (problem: string, extra: Partial<SnapshotCheck> = {}): SnapshotCheck => ({ ok: false, integrity: null, migrations: null, appMigrations, newerThanApp: false, counts: null, problem, ...extra });
if (!existsSync(file)) return fail("The file isn't there.");
if (!hasSqliteHeader(file)) return fail("That isn't a database file.");
const c = createClient({ url: `file:${file}` });
try {
const one = async (sql: string) => Object.values((await c.execute(sql)).rows[0] ?? {})[0];
const integrity = String(await one("PRAGMA integrity_check"));
const tables = new Set((await c.execute("select name from sqlite_master where type = 'table'")).rows.map((r) => String(r.name)));
for (const needed of ["users", "servers", "integrations", "settings", "__drizzle_migrations"]) {
if (!tables.has(needed)) return fail(`It isn't a Homelab Manager database (no "${needed}" table).`, { integrity });
}
const migrations = Number(await one("select count(*) from __drizzle_migrations"));
const counts = { users: Number(await one("select count(*) from users")), servers: Number(await one("select count(*) from servers")), integrations: Number(await one("select count(*) from integrations")) };
const newerThanApp = appMigrations > 0 && migrations > appMigrations;
if (integrity !== "ok") return fail(`SQLite found problems in it: ${integrity}`, { integrity, migrations, counts, newerThanApp });
if (newerThanApp) return fail("It was made by a newer version of Homelab Manager than this one, and an older version can't use it. Update first.", { integrity, migrations, counts, newerThanApp });
return { ok: true, integrity, migrations, appMigrations, newerThanApp: false, counts, problem: null };
} catch (err) {
return fail(`It couldn't be read as a database: ${err instanceof Error ? err.message : String(err)}`);
} finally {
c.close();
}
}
/** A name for a new file that isn't taken. */
function freshName(kind: SnapshotKind, now: Date): string {
const base = `homelab-manager-${stamp(now)}${kind === "scheduled" ? "" : `-${kind}`}`;
let name = `${base}.sqlite`;
for (let n = 2; existsSync(join(backupDir(), name)); n++) name = `${base}-${n}.sqlite`;
return name;
}
export interface LastAttempt {
at: string;
ok: boolean;
kind: SnapshotKind;
error: string | null;
name: string | null;
}
const ATTEMPT_FLAG = "selfBackupLastAttempt";
export async function lastAttempt(): Promise<LastAttempt | null> {
try {
const v = await getInternalFlag(ATTEMPT_FLAG);
return v ? (JSON.parse(v) as LastAttempt) : null;
} catch {
return null;
}
}
/** Writes a consistent copy of the database into the backups folder and checks it. Throws, saying why, if it can't — and records the attempt either way. */
export async function createSnapshot(kind: SnapshotKind = "manual", now: Date = new Date()): Promise<{ info: SnapshotInfo; check: SnapshotCheck }> {
const tmp = scratch();
try {
mkdirSync(backupDir(), { recursive: true });
const name = freshName(kind, now);
const file = join(backupDir(), name);
// Made and checked in a temporary folder first, and only then put in the backups folder: what is there is always whole and sound.
await client.execute(`VACUUM INTO '${tmp.file.replace(/'/g, "''")}'`);
const check = await checkOpened(tmp.file);
if (!check.ok) throw new Error(`The copy didn't check out, so it was thrown away: ${check.problem}`);
copyFileSync(tmp.file, file);
try {
chmodSync(file, 0o600);
} catch {
// not every file system has modes
}
const info: SnapshotInfo = { name, kind, sizeBytes: statSync(file).size, at: now.toISOString() };
await setInternalFlag(ATTEMPT_FLAG, JSON.stringify({ at: now.toISOString(), ok: true, kind, error: null, name } satisfies LastAttempt));
return { info, check };
} catch (err) {
const error = err instanceof Error ? err.message : String(err);
await setInternalFlag(ATTEMPT_FLAG, JSON.stringify({ at: now.toISOString(), ok: false, kind, error, name: null } satisfies LastAttempt));
throw err;
} finally {
tmp.done();
}
}
export function listSnapshots(): SnapshotInfo[] {
if (!existsSync(backupDir())) return [];
return readdirSync(backupDir())
.filter(isSnapshotName)
.map((name) => {
const st = statSync(join(backupDir(), name));
return { name, kind: kindOf(name), sizeBytes: st.size, at: st.mtime.toISOString() };
})
.sort((a, b) => b.at.localeCompare(a.at) || b.name.localeCompare(a.name));
}
/** What is kept: the newest `keep` scheduled backups, and a few of each other kind (a restore's "before" copy is worth keeping longer than a day). */
export const KEEP_OTHER: Record<Exclude<SnapshotKind, "scheduled">, number> = { manual: 10, "pre-restore": 3, uploaded: 3 };
export function pruneSnapshots(keep: number): string[] {
const removed: string[] = [];
const all = listSnapshots();
const limitFor = (kind: SnapshotKind) => (kind === "scheduled" ? Math.max(1, keep) : KEEP_OTHER[kind]);
for (const kind of ["scheduled", "manual", "pre-restore", "uploaded"] as const) {
for (const s of all.filter((x) => x.kind === kind).slice(limitFor(kind))) {
rmSync(join(backupDir(), s.name), { force: true });
removed.push(s.name);
}
}
return removed;
}
export function snapshotPath(name: string): string | null {
if (!isSnapshotName(name)) return null;
const file = join(backupDir(), name);
return existsSync(file) ? file : null;
}
export function deleteSnapshot(name: string): boolean {
const file = snapshotPath(name);
if (!file) return false;
rmSync(file, { force: true });
return true;
}
/** A file somebody uploaded, kept as a backup if it checks out (and thrown away if not). */
export async function importUploaded(bytes: Buffer, now: Date = new Date()): Promise<{ info: SnapshotInfo; check: SnapshotCheck }> {
const tmp = scratch();
try {
writeFileSync(tmp.file, bytes);
const check = await checkOpened(tmp.file);
if (!check.ok) throw new Error(check.problem ?? "That file isn't usable.");
mkdirSync(backupDir(), { recursive: true });
const name = freshName("uploaded", now);
const file = join(backupDir(), name);
copyFileSync(tmp.file, file);
try {
chmodSync(file, 0o600);
} catch {
// not every file system has modes
}
return { info: { name, kind: "uploaded", sizeBytes: statSync(file).size, at: now.toISOString() }, check };
} finally {
tmp.done();
}
}
// ─── Restoring ──────────────────────────────────────────────────────────────
export interface StagedRestore {
from: string;
stagedAt: string;
by: string | null;
}
export function stagedRestore(): StagedRestore | null {
try {
if (!existsSync(join(dataDir(), PENDING_FILE))) return null;
const note = JSON.parse(readFileSync(join(dataDir(), PENDING_NOTE), "utf8")) as { from?: string; stagedAt?: string; by?: string | null };
return { from: note.from ?? "a backup", stagedAt: note.stagedAt ?? "", by: note.by ?? null };
} catch {
return existsSync(join(dataDir(), PENDING_FILE)) ? { from: "a backup", stagedAt: "", by: null } : null;
}
}
/** Checks a backup and stages it to replace the database the next time the app starts. Throws, saying why, if it can't be used. */
export async function stageRestore(name: string, by: string | null, now: Date = new Date()): Promise<SnapshotCheck> {
const file = snapshotPath(name);
if (!file) throw new Error("There is no backup with that name.");
const check = await checkSnapshotFile(file);
if (!check.ok) throw new Error(check.problem ?? "That backup isn't usable.");
const pending = join(dataDir(), PENDING_FILE);
const tmp = `${pending}.copying`;
copyFileSync(file, tmp);
renameSync(tmp, pending); // whole, or not there at all: the start-up step never sees half a file
try {
chmodSync(pending, 0o600);
} catch {
// not every file system has modes
}
writeFileSync(join(dataDir(), PENDING_NOTE), JSON.stringify({ from: name, stagedAt: now.toISOString(), by }));
return check;
}
export function cancelStagedRestore(): boolean {
const had = existsSync(join(dataDir(), PENDING_FILE));
rmSync(join(dataDir(), PENDING_FILE), { force: true });
rmSync(join(dataDir(), PENDING_NOTE), { force: true });
return had;
}
/** What the last restore did, once, for the page to say. */
export function lastRestore(): RestoreResult | null {
try {
return JSON.parse(readFileSync(join(dataDir(), RESULT_NOTE), "utf8")) as RestoreResult;
} catch {
return null;
}
}
export function dismissLastRestore(): void {
rmSync(join(dataDir(), RESULT_NOTE), { force: true });
}
// ─── How current the backups are ────────────────────────────────────────────
/** Backups older than this are an alert: a day's slack on top of the daily schedule. */
export const STALE_AFTER_HOURS = 50;
export function newestSnapshot(): SnapshotInfo | null {
return listSnapshots().find((s) => s.kind === "scheduled" || s.kind === "manual") ?? null;
}
@@ -0,0 +1,65 @@
import schedule from "node-schedule";
import { getSettings } from "./settingsStore.js";
import { createSnapshot, lastAttempt, newestSnapshot, pruneSnapshots } from "./selfBackup.js";
import { notify } from "./notify.js";
let currentJob: schedule.Job | null = null;
/** The cron rule for "every day at HH:MM". */
export function cronForBackup(time: string): string {
const [h, m] = time.split(":").map(Number);
return `${Number.isFinite(m) ? m : 30} ${Number.isFinite(h) ? h : 3} * * *`;
}
/** One scheduled backup: take it, then drop the oldest beyond how many are kept. Says so through the channels if it fails. */
export async function runScheduledBackup(now: Date = new Date()): Promise<{ name: string } | null> {
const { selfBackup, notifications } = await getSettings();
try {
const { info } = await createSnapshot("scheduled", now);
pruneSnapshots(selfBackup.keep);
return { name: info.name };
} catch (err) {
console.error("[selfBackup] the scheduled backup failed:", err instanceof Error ? err.message : err);
if (notifications.healthAlerts !== false) {
await notify("Homelab Manager — its own backup failed", `The scheduled backup of Homelab Manager's database didn't work: ${err instanceof Error ? err.message : String(err)}`).catch(() => {});
}
return null;
}
}
/** The next time it will run, or null when it is off. */
export function nextBackupRun(): Date | null {
const next = currentJob?.nextInvocation();
return next ? new Date(next.getTime()) : null;
}
/** (Re)schedules the daily backup per the current settings. Call again after they change. */
export async function scheduleSelfBackup(): Promise<void> {
if (currentJob) {
currentJob.cancel();
currentJob = null;
}
const { selfBackup, notifications } = await getSettings();
if (!selfBackup.enabled) {
console.log("[selfBackup] daily backups are off, nothing scheduled");
return;
}
currentJob = schedule.scheduleJob({ rule: cronForBackup(selfBackup.time), tz: notifications.timezone }, () => {
void runScheduledBackup();
});
console.log(`Daily backup scheduled at ${selfBackup.time} (${notifications.timezone})`);
}
/** Schedules it, and takes one soon if the app has been off through its time (or has never taken one). */
export async function initSelfBackupScheduler(): Promise<void> {
await scheduleSelfBackup();
const { selfBackup } = await getSettings();
if (!selfBackup.enabled) return;
const newest = newestSnapshot();
const overdue = !newest || Date.now() - Date.parse(newest.at) > 26 * 3600_000;
// Only retry a failing one at the next scheduled time, not at every restart.
const last = await lastAttempt();
if (overdue && !(last && !last.ok && Date.now() - Date.parse(last.at) < 6 * 3600_000)) {
setTimeout(() => void runScheduledBackup(), 30_000).unref();
}
}
+11
View File
@@ -72,6 +72,17 @@ export function describeAudit(category: string, action: string, detail: Record<s
return { title: "Admin link removed", tone: "neutral", kind: "change", detail: label };
}
}
if (category === "alert") {
const note = typeof detail.note === "string" && detail.note ? detail.note : null;
const assignee = typeof detail.assignee === "string" ? detail.assignee : null;
const lasts = typeof detail.hours === "number" ? `for ${detail.hours >= 48 ? `${Math.round(detail.hours / 24)} days` : `${detail.hours} h`}` : "until the alert is gone";
switch (action) {
case "ack":
return { title: assignee ? `Acknowledged, handed to ${assignee}` : "Acknowledged", tone: "info", kind: "change", detail: [note, lasts].filter(Boolean).join(" · ") };
case "unack":
return { title: "Acknowledgement taken back", tone: "neutral", kind: "change", detail: null };
}
}
if (category === "update_run") {
const template = typeof detail.template === "string" ? detail.template : null;
switch (action) {
+11
View File
@@ -129,6 +129,15 @@ export interface AgentSettings {
autoUpdate: boolean;
}
/** The daily backup of Homelab Manager's own database (see services/selfBackup.ts). On unless an administrator turns it off. */
export interface SelfBackupSettings {
enabled: boolean;
/** "HH:MM", in the notification time zone. */
time: string;
/** How many of the daily backups are kept; the oldest beyond that are deleted. */
keep: number;
}
/** What the inventory needs to work out a running cost. Without a price per kWh it shows draw but no cost. */
export interface InventorySettings {
/** A short name shown after amounts ("SEK", "kr", "$"); may be blank. */
@@ -190,6 +199,7 @@ export interface AppSettings {
digest: DigestSettings;
signIn: SignInSettings;
inventory: InventorySettings;
selfBackup: SelfBackupSettings;
agents: AgentSettings;
consistency: ConsistencySettings;
nameGenerator: NameGeneratorSettings;
@@ -227,6 +237,7 @@ const DEFAULTS: AppSettings = {
digest: { enabled: false, day: 1, time: "08:00" },
inventory: { currency: "", pricePerKwh: null },
agents: { autoUpdate: false },
selfBackup: { enabled: true, time: "03:30", keep: 14 },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
+21
View File
@@ -20,6 +20,8 @@ import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
import { attentionCounts, loadAcks } from "./alertAcks.js";
import { listIncidents } from "./incidents.js";
export const DIGEST_DAYS = 7;
const DAY_MS = 24 * 3600_000;
@@ -90,11 +92,30 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
if (report.counts.silenced > 0) held.push(`${report.counts.silenced} silenced by maintenance`);
if (report.counts.grouped > 0) held.push(`${report.counts.grouped} folded under a server that is down`);
if (held.length > 0) lines.push(`(${held.join(", ")})`);
const acknowledged = attentionCounts(report.alerts, await loadAcks(now)).acknowledged;
if (acknowledged > 0) lines.push(`(${acknowledged} of them acknowledged — somebody is on it)`);
sections.push({ heading: "Right now", lines, attention: active.length > 0 || report.couldntCheck.length > 0 });
} catch (err) {
console.error("[digest] couldn't read the alerts:", err);
}
// ── Incidents ──
try {
const since = now.getTime() - DIGEST_DAYS * DAY_MS;
const all = await listIncidents({ now });
const begun = all.filter((i) => Date.parse(i.openedAt) >= since);
const ended = all.filter((i) => i.resolvedAt !== null && Date.parse(i.resolvedAt) >= since);
const open = all.filter((i) => i.open);
if (begun.length > 0 || ended.length > 0) {
const lines = [`${plural(begun.length, "incident")} began, ${ended.length} ended, ${open.length} still open.`];
const longest = [...ended].sort((a, b) => b.durationMs - a.durationMs).slice(0, 3);
lines.push(...longest.map((i) => `- ${i.message} — lasted ${spanWords(i.durationMs / 60_000)}`));
sections.push({ heading: "Incidents", lines, attention: open.length > 0 });
}
} catch (err) {
console.error("[digest] couldn't read the incidents:", err);
}
// ── Uptime ──
try {
const uptime = await getUptime(now, DIGEST_DAYS);