Acknowledge and hand over alerts, incident timelines, and built-in database backups

- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what
  needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics
- Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents,
  incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs,
  related alerts and notes; Incidents page and a digest section
- Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download,
  check, upload, delete, and restore staged and applied at start-up before the database opens
  (pre-restore copy kept); alerts and /metrics for stale or failing backups
- Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 02:21:08 +02:00
1 parent 7a5b623444
commit 2744a84719
41 files changed
+6573 -26

No files matched your search

@@ -0,0 +1,46 @@
CREATE TABLE `alert_acks` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`alert_key` text NOT NULL,
`note` text DEFAULT '' NOT NULL,
`assignee_user_id` integer,
`assignee_name` text,
`acked_by` text NOT NULL,
`acked_at` text NOT NULL,
`until` text,
`last_seen_at` text NOT NULL,
FOREIGN KEY (`assignee_user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE set null
);
--> statement-breakpoint
CREATE UNIQUE INDEX `alert_acks_key_idx` ON `alert_acks` (`alert_key`);--> statement-breakpoint
CREATE TABLE `alert_incidents` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`alert_key` text NOT NULL,
`severity` text NOT NULL,
`peak_severity` text NOT NULL,
`category` text NOT NULL,
`source` text NOT NULL,
`message` text NOT NULL,
`link` text,
`target_type` text,
`target_id` integer,
`target_name` text,
`caused_by` text,
`opened_at` text NOT NULL,
`resolved_at` text,
`last_seen_at` text NOT NULL,
`missing_since` text
);
--> statement-breakpoint
CREATE INDEX `alert_incidents_key_idx` ON `alert_incidents` (`alert_key`);--> statement-breakpoint
CREATE INDEX `alert_incidents_opened_idx` ON `alert_incidents` (`opened_at`);--> statement-breakpoint
CREATE INDEX `alert_incidents_target_idx` ON `alert_incidents` (`target_type`,`target_id`);--> statement-breakpoint
CREATE TABLE `incident_notes` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`incident_id` integer NOT NULL,
`author` text NOT NULL,
`body` text NOT NULL,
`created_at` text NOT NULL,
FOREIGN KEY (`incident_id`) REFERENCES `alert_incidents`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `incident_notes_incident_idx` ON `incident_notes` (`incident_id`);
File diff suppressed because it is too large. Load diff
+7
View File
@@ -225,6 +225,13 @@
"when": 1792278718505,
"tag": "0031_sad_lizard",
"breakpoints": true
},
{
"idx": 32,
"version": "6",
"when": 1792281947686,
"tag": "0032_daffy_captain_stacy",
"breakpoints": true
}
]
}
+4
View File
@@ -12,6 +12,8 @@ import { uptimeRouter } from "./routes/uptime.js";
import { shareLinksRouter } from "./routes/shareLinks.js";
import { runbooksRouter } from "./routes/runbooks.js";
import { updateRunsRouter } from "./routes/updateRuns.js";
import { dbBackupsRouter } from "./routes/dbBackups.js";
import { incidentsRouter } from "./routes/incidents.js";
import { inventoryRouter } from "./routes/inventory.js";
import { backupsRouter } from "./routes/backups.js";
import { metricsRouter, metricsTokensRouter } from "./routes/metrics.js";
@@ -100,6 +102,8 @@ export function createApp() {
app.use("/api/share-links", shareLinksRouter);
app.use("/api/runbooks", runbooksRouter);
app.use("/api/update-runs", updateRunsRouter);
app.use("/api/db-backups", dbBackupsRouter);
app.use("/api/incidents", incidentsRouter);
app.use("/api/inventory", inventoryRouter);
app.use("/api/backups", backupsRouter);
app.use("/api/metrics-tokens", metricsTokensRouter);
+71
View File
@@ -0,0 +1,71 @@
/**
* Finishes a restore that was staged from Settings → Backup. It has to happen before the database is opened, which is why it is a module
* of its own that index.ts imports first: by the time anything else loads, the file is already the one that was chosen.
*
* Nothing is deleted. The database that was in use is moved into the backups folder (as "…-pre-restore.sqlite") before the chosen one is put
* in its place, so a restore that wasn't what was wanted can itself be undone by restoring that file. If anything goes wrong half way, the
* old file is put back and the staged one is left where it is.
*/
import "../loadEnv.js";
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { resolveDataPath } from "../paths.js";
import { PENDING_FILE, PENDING_NOTE, RESULT_NOTE, type RestoreResult } from "./restoreFiles.js";
const stamp = (d: Date) => d.toISOString().slice(0, 19).replace(/[:T]/g, "-");
/** Puts a staged database in place, if one is waiting. Returns what it did, or null if nothing was staged. */
export function applyPendingRestore(dbFile: string, now: Date = new Date()): RestoreResult | null {
const dataDir = dirname(dbFile);
const pending = join(dataDir, PENDING_FILE);
if (!existsSync(pending)) return null;
// Only a file is ever put in place of the database.
if (!statSync(pending).isFile()) {
console.error(`[restore] ${PENDING_FILE} isn't a file, so it was left alone and the database is as it was.`);
return null;
}
let note: { from?: string; by?: string | null } = {};
try {
note = JSON.parse(readFileSync(join(dataDir, PENDING_NOTE), "utf8"));
} catch {
// the note only says where it came from; the file is what matters
}
const backups = join(dataDir, "backups");
let keptAs: string | null = null;
let moved = false;
try {
mkdirSync(backups, { recursive: true });
if (existsSync(dbFile)) {
let name = `homelab-manager-${stamp(now)}-pre-restore.sqlite`;
for (let n = 2; existsSync(join(backups, name)); n++) name = `homelab-manager-${stamp(now)}-pre-restore-${n}.sqlite`;
renameSync(dbFile, join(backups, name));
keptAs = name;
moved = true;
}
// A journal left by the old file would be taken for the new one's.
for (const suffix of ["-wal", "-shm", "-journal"]) rmSync(dbFile + suffix, { force: true });
renameSync(pending, dbFile);
} catch (err) {
if (moved && keptAs && !existsSync(dbFile)) {
try {
renameSync(join(backups, keptAs), dbFile);
} catch {
// nothing more can be done; the old file is still in the backups folder under its new name
}
}
console.error("[restore] couldn't put the staged backup in place; the database is as it was:", err instanceof Error ? err.message : err);
return null;
}
const result: RestoreResult = { restoredAt: now.toISOString(), from: note.from ?? "an uploaded backup", keptAs, by: note.by ?? null };
writeFileSync(join(dataDir, RESULT_NOTE), JSON.stringify(result));
rmSync(join(dataDir, PENDING_NOTE), { force: true });
console.log(`[restore] restored the database from ${result.from}${keptAs ? `; the one that was in use is kept as backups/${keptAs}` : ""}.`);
return result;
}
// Run it now, on the database this process is about to open.
applyPendingRestore(resolveDataPath(process.env.DATABASE_PATH ?? "../data/homelab-manager.sqlite"));
+1 -1
View File
@@ -12,4 +12,4 @@ const client = createClient({ url: `file:${dbPath}` });
await client.execute("PRAGMA foreign_keys = ON;");
export const db = drizzle(client, { schema });
export { client };
export { client, dbPath };
+13
View File
@@ -0,0 +1,13 @@
// The names a staged restore uses in the data folder. Kept apart from applyRestore.ts, which does something when it is imported.
export const PENDING_FILE = "restore-pending.sqlite";
export const PENDING_NOTE = "restore-pending.json";
export const RESULT_NOTE = "restore-result.json";
export interface RestoreResult {
restoredAt: string;
/** The backup that is now the database. */
from: string;
/** Where the database that was in use went. */
keptAs: string | null;
by: string | null;
}
+57
View File
@@ -572,6 +572,63 @@ export const runbooks = sqliteTable(
(t) => [index("runbooks_target_idx").on(t.targetType, t.targetId)],
);
// ─── Acknowledgements and incidents ─────────────────────────────────────────
// "I'm on it" for one alert — see services/alertAcks.ts. Keyed by the alert's stable id (not a foreign key to anything); gone once the alert has been.
export const alertAcks = sqliteTable(
"alert_acks",
{
id: integer("id").primaryKey({ autoIncrement: true }),
alertKey: text("alert_key").notNull(),
note: text("note").notNull().default(""),
assigneeUserId: integer("assignee_user_id").references(() => users.id, { onDelete: "set null" }),
assigneeName: text("assignee_name"), // as it was when handed over, so it still reads if the user is removed
ackedBy: text("acked_by").notNull(),
ackedAt: text("acked_at").notNull(), // ISO
until: text("until"), // ISO; null = until the alert is gone
lastSeenAt: text("last_seen_at").notNull(), // ISO; the last time the alert was there
},
(t) => [uniqueIndex("alert_acks_key_idx").on(t.alertKey)],
);
// One time an alert was raised and later went away — see services/incidents.ts. When the same alert comes back it is a new row.
export const alertIncidents = sqliteTable(
"alert_incidents",
{
id: integer("id").primaryKey({ autoIncrement: true }),
alertKey: text("alert_key").notNull(),
severity: text("severity").notNull(), // as it is now (or was when it ended)
peakSeverity: text("peak_severity").notNull(),
category: text("category").notNull(),
source: text("source").notNull(),
message: text("message").notNull(), // the latest wording
link: text("link"),
targetType: text("target_type"), // server | integration | dns_provider, when the alert is about one of those
targetId: integer("target_id"),
targetName: text("target_name"),
causedBy: text("caused_by"), // the host whose outage this was folded under
openedAt: text("opened_at").notNull(), // ISO
resolvedAt: text("resolved_at"), // ISO; null while it is open
lastSeenAt: text("last_seen_at").notNull(), // ISO
missingSince: text("missing_since"), // ISO; set while it has been absent from the last look(s), before it is called over
},
(t) => [index("alert_incidents_key_idx").on(t.alertKey), index("alert_incidents_opened_idx").on(t.openedAt), index("alert_incidents_target_idx").on(t.targetType, t.targetId)],
);
export const incidentNotes = sqliteTable(
"incident_notes",
{
id: integer("id").primaryKey({ autoIncrement: true }),
incidentId: integer("incident_id")
.notNull()
.references(() => alertIncidents.id, { onDelete: "cascade" }),
author: text("author").notNull(),
body: text("body").notNull(),
createdAt: text("created_at").notNull(), // ISO
},
(t) => [index("incident_notes_incident_idx").on(t.incidentId)],
);
// ─── Update runs: apply a server's updates and see it through ──────────────
// A playbook run in Semaphore for one server, wrapped in a maintenance window and followed until the server reports again — see
+7
View File
@@ -1,4 +1,6 @@
import "./loadEnv.js";
// Before anything opens the database: puts a restore that was staged from Settings → Backup in place.
import "./db/applyRestore.js";
import { env, warnIfAuthNotConfigured, checkSessionSecret } from "./env.js";
import { createApp } from "./app.js";
import { runMigrations } from "./db/migrate.js";
@@ -17,6 +19,9 @@ import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceScheduler.js";
import { initMaintenanceScheduleScheduler } from "./services/maintenanceScheduleScheduler.js";
import { initUpdateRuns } from "./services/updateRuns.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
import { initIncidentRecorder } from "./services/incidents.js";
import { getAlerts } from "./services/alerts.js";
{
// Sessions are signed with this secret. Without one, the app would sign them with a value that's in the source code.
@@ -50,6 +55,8 @@ await initAlertTicketScheduler();
await initKumaMaintenanceScheduler();
await initMaintenanceScheduleScheduler();
initUpdateRuns();
await initSelfBackupScheduler();
initIncidentRecorder(() => getAlerts(false));
createApp().listen(env.port, () => {
console.log(`homelab-manager listening on port ${env.port}`);
+69 -2
View File
@@ -1,6 +1,10 @@
import { Router } from "express";
import { requireAuth } from "../auth/middleware.js";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { getAlerts } from "../services/alerts.js";
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
import { recordAudit } from "../services/audit.js";
import { openIncidentIds } from "../services/incidents.js";
import { ticketsForAlerts, type AlertTicketInfo } from "../services/alertTickets.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -19,5 +23,68 @@ alertsRouter.get("/", asyncHandler(async (req, res) => {
} catch (err) {
console.error("[alerts] couldn't look up tickets:", err);
}
res.json({ ...report, alerts: report.alerts.map((a) => ({ ...a, ticket: tickets.get(a.id) ?? null })) });
// Who has said they are on it, and the incident record each alert is part of. Neither may stop the page from loading.
let acks = new Map<string, Awaited<ReturnType<typeof loadAcks>> extends Map<string, infer V> ? V : never>();
let incidents = new Map<string, number>();
try {
acks = await loadAcks();
incidents = await openIncidentIds(report.alerts.map((a) => a.id));
} catch (err) {
console.error("[alerts] couldn't look up acknowledgements:", err);
}
const attention = attentionCounts(report.alerts, acks);
res.json({
...report,
// `counts` still counts everything, as /metrics and the wall do; `attention` leaves out what somebody has said they are dealing with.
counts: { ...report.counts, acknowledged: attention.acknowledged, attention: { critical: attention.critical, warning: attention.warning } },
alerts: report.alerts.map((a) => ({ ...a, ticket: tickets.get(a.id) ?? null, ack: acks.get(a.id) ?? null, incidentId: incidents.get(a.id) ?? null })),
});
}));
// Who an alert can be handed to.
alertsRouter.get("/assignees", requireRole("operator"), asyncHandler(async (_req, res) => {
res.json({ users: await assignableUsers() });
}));
const ackSchema = z.object({
alertId: z.string().min(1).max(600),
note: z.string().max(MAX_NOTE).default(""),
assigneeId: z.number().int().positive().nullable().default(null),
/** How long it lasts; null (or left out) is until the alert is gone. */
hours: z.number().int().min(1).max(MAX_HOURS).nullable().default(null),
});
const auditTargetOf = (alert: { target: { type: string; id: number } | null }) => (alert.target ? { targetType: alert.target.type, targetId: alert.target.id } : {});
// "I'm on it": acknowledge an alert, say why, hand it to someone, and say how long it should be left alone.
alertsRouter.put("/ack", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = ackSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "Say which alert, and optionally why, who has it and for how long.", details: parsed.error.flatten() });
const alert = (await getAlerts(false)).alerts.find((a) => a.id === parsed.data.alertId);
if (!alert) return res.status(404).json({ error: "not_found", message: "That alert isn't there any more." });
const actor = req.currentUser!;
try {
const { ack, before } = await setAck({ ...parsed.data, actor: actor.name ?? actor.email ?? actor.oidcSub });
await recordAudit({
actor,
category: "alert",
action: "ack",
...auditTargetOf(alert),
detail: { alertId: alert.id, message: alert.message, note: ack.note, assignee: ack.assignee?.name ?? null, hours: parsed.data.hours, changed: before !== null },
});
res.json({ ack });
} catch (err) {
if (err instanceof AckError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
alertsRouter.delete("/ack", requireRole("operator"), asyncHandler(async (req, res) => {
const alertId = typeof req.query.alertId === "string" ? req.query.alertId : "";
if (!alertId) return res.status(400).json({ error: "invalid_id" });
const removed = await clearAck(alertId);
if (!removed) return res.status(404).json({ error: "not_found" });
const alert = (await getAlerts(false)).alerts.find((a) => a.id === alertId);
await recordAudit({ actor: req.currentUser!, category: "alert", action: "unack", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: alert?.message ?? null } });
res.status(204).end();
}));
+135
View File
@@ -0,0 +1,135 @@
import express, { Router } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
import {
backupDir,
cancelStagedRestore,
checkSnapshotFile,
createSnapshot,
deleteSnapshot,
dismissLastRestore,
importUploaded,
lastAttempt,
lastRestore,
listSnapshots,
pruneSnapshots,
snapshotPath,
stageRestore,
stagedRestore,
} from "../services/selfBackup.js";
import { nextBackupRun, scheduleSelfBackup } from "../services/selfBackupScheduler.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const dbBackupsRouter = Router();
// A backup holds everything the app knows — the audit log, every server's address, the hashed agent tokens and all the stored credentials
// (encrypted) — so all of it, looking and doing, is for administrators.
dbBackupsRouter.use(requireAuth, requireRole("admin"));
/** A database file is far bigger than anything else this app is sent, and is only ever read from an administrator. */
const UPLOAD_LIMIT = "1gb";
const settingsSchema = z.object({
enabled: z.boolean(),
time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/, "Use a time like 03:30"),
keep: z.number().int().min(1).max(90),
});
const actorName = (req: express.Request) => req.currentUser!.name ?? req.currentUser!.email ?? req.currentUser!.oidcSub;
async function overview() {
const { selfBackup } = await getSettings();
return {
settings: selfBackup,
snapshots: listSnapshots(),
staged: stagedRestore(),
lastRestore: lastRestore(),
lastAttempt: await lastAttempt(),
nextRun: nextBackupRun()?.toISOString() ?? null,
folder: backupDir(),
};
}
dbBackupsRouter.get("/", asyncHandler(async (_req, res) => {
res.json(await overview());
}));
dbBackupsRouter.put("/settings", asyncHandler(async (req, res) => {
const parsed = settingsSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "A time and how many to keep.", details: parsed.error.flatten() });
const before = (await getSettings()).selfBackup;
await updateSettings({ selfBackup: parsed.data });
await scheduleSelfBackup();
pruneSnapshots(parsed.data.keep);
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "settings", targetType: "settings", detail: { from: before, to: parsed.data } });
res.json(await overview());
}));
// Take one now. It is checked as it is made, and pruned like the scheduled ones.
dbBackupsRouter.post("/", asyncHandler(async (req, res) => {
try {
const { info, check } = await createSnapshot("manual");
pruneSnapshots((await getSettings()).selfBackup.keep);
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "create", targetType: "db_backup", targetId: info.name, detail: { sizeBytes: info.sizeBytes } });
res.status(201).json({ snapshot: info, check });
} catch (err) {
res.status(500).json({ error: "backup_failed", message: err instanceof Error ? err.message : String(err) });
}
}));
// A backup somebody has (from another machine, from the script in BACKUP.md): kept if it checks out, and can then be restored like any other.
dbBackupsRouter.post("/upload", express.raw({ type: () => true, limit: UPLOAD_LIMIT }), asyncHandler(async (req, res) => {
if (!Buffer.isBuffer(req.body) || req.body.length === 0) return res.status(400).json({ error: "no_file", message: "Choose a backup file to upload." });
try {
const { info, check } = await importUploaded(req.body);
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "upload", targetType: "db_backup", targetId: info.name, detail: { sizeBytes: info.sizeBytes } });
res.status(201).json({ snapshot: info, check });
} catch (err) {
res.status(400).json({ error: "unusable", message: err instanceof Error ? err.message : String(err) });
}
}));
dbBackupsRouter.post("/restore/cancel", asyncHandler(async (req, res) => {
const had = cancelStagedRestore();
if (had) await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "restore_cancelled" });
res.json({ cancelled: had });
}));
dbBackupsRouter.post("/restore-result/dismiss", asyncHandler(async (_req, res) => {
dismissLastRestore();
res.json({ ok: true });
}));
dbBackupsRouter.post("/:name/check", asyncHandler(async (req, res) => {
const file = snapshotPath(req.params.name);
if (!file) return res.status(404).json({ error: "not_found" });
res.json({ check: await checkSnapshotFile(file) });
}));
dbBackupsRouter.get("/:name/download", asyncHandler(async (req, res) => {
const file = snapshotPath(req.params.name);
if (!file) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "download", targetType: "db_backup", targetId: req.params.name });
res.setHeader("Cache-Control", "no-store");
res.download(file, req.params.name);
}));
// Checks the backup and stages it to replace the database the next time the app starts. Nothing changes until then, and the database in
// use is kept (as a "pre-restore" backup) when it does.
dbBackupsRouter.post("/:name/restore", asyncHandler(async (req, res) => {
try {
const check = await stageRestore(req.params.name, actorName(req));
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "restore_staged", targetType: "db_backup", targetId: req.params.name });
res.json({ staged: stagedRestore(), check });
} catch (err) {
res.status(400).json({ error: "cannot_restore", message: err instanceof Error ? err.message : String(err) });
}
}));
dbBackupsRouter.delete("/:name", asyncHandler(async (req, res) => {
if (!deleteSnapshot(req.params.name)) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "delete", targetType: "db_backup", targetId: req.params.name });
res.status(204).end();
}));
+51
View File
@@ -0,0 +1,51 @@
import { Router } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { addNote, getIncident, listIncidents, MAX_NOTE, removeNote } from "../services/incidents.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const incidentsRouter = Router();
// Everyone signed in can read what happened — it is made of what the Alerts page, the server pages and the Maintenance page already show.
// Writing a note is for operators.
incidentsRouter.use(requireAuth);
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
incidentsRouter.get("/", asyncHandler(async (_req, res) => {
res.json({ incidents: await listIncidents() });
}));
incidentsRouter.get("/:id", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const detail = await getIncident(id);
if (!detail) return res.status(404).json({ error: "not_found" });
res.json(detail);
}));
const noteSchema = z.object({ body: z.string().trim().min(1, "Write something first.").max(MAX_NOTE) });
incidentsRouter.post("/:id/notes", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = noteSchema.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.success ? undefined : parsed.error.issues[0]?.message });
const actor = req.currentUser!;
const note = await addNote(id, actor.name ?? actor.email ?? actor.oidcSub, parsed.data.body);
if (!note) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor, category: "incident", action: "note", targetType: "incident", targetId: id });
res.status(201).json({ id: note.id });
}));
incidentsRouter.delete("/:id/notes/:noteId", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const noteId = idOf(req.params.noteId);
if (!Number.isInteger(id) || !Number.isInteger(noteId)) return res.status(400).json({ error: "invalid_id" });
const actor = req.currentUser!;
const outcome = await removeNote(id, noteId, { name: actor.name ?? actor.email ?? actor.oidcSub, isAdmin: actor.role === "admin" });
if (outcome === "not_found") return res.status(404).json({ error: "not_found" });
if (outcome === "forbidden") return res.status(403).json({ error: "forbidden", message: "A note can be removed by the person who wrote it, or an administrator." });
await recordAudit({ actor, category: "incident", action: "note_removed", targetType: "incident", targetId: id });
res.status(204).end();
}));
+133
View File
@@ -0,0 +1,133 @@
/**
* "I'm on it": an acknowledgement says that somebody has seen an alert, may say who is dealing with it and why, and may run out after a
* while. It changes nothing about the problem — the alert is still raised, still counted by /metrics and still on the wall display — but
* the people looking at the app can tell what nobody has looked at yet from what somebody has, and the sidebar and dashboard count only the
* former.
*
* An acknowledgement belongs to the alert's id (the same stable key the Alerts page keys rows on). It goes away on its own once the alert
* has been gone for a while, so the same problem coming back later is a new alert that nobody has looked at, not one that is "still being
* dealt with" from last month.
*/
import { and, eq, inArray, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { alertAcks, users } from "../db/schema.js";
import type { Alert } from "./alertTypes.js";
export interface AckInfo {
note: string;
assignee: { id: number | null; name: string } | null;
ackedBy: string;
ackedAt: string;
/** ISO — when it runs out; null means it lasts until the alert is gone. */
until: string | null;
/** Still in force: has no end, or the end hasn't come. */
active: boolean;
}
/** An alert that has been absent this long loses its acknowledgement (a blip of a few minutes doesn't). */
export const GONE_AFTER_MS = 30 * 60_000;
export const MAX_NOTE = 500;
export const MAX_HOURS = 24 * 30;
export class AckError extends Error {
constructor(
readonly status: number,
readonly code: string,
message: string,
) {
super(message);
}
}
const shape = (r: typeof alertAcks.$inferSelect, now: Date): AckInfo => ({
note: r.note,
assignee: r.assigneeName ? { id: r.assigneeUserId, name: r.assigneeName } : null,
ackedBy: r.ackedBy,
ackedAt: r.ackedAt,
until: r.until,
active: r.until === null || Date.parse(r.until) > now.getTime(),
});
export async function loadAcks(now: Date = new Date()): Promise<Map<string, AckInfo>> {
const rows = await db.select().from(alertAcks);
return new Map(rows.map((r) => [r.alertKey, shape(r, now)]));
}
export async function loadAck(alertKey: string, now: Date = new Date()): Promise<AckInfo | null> {
const [row] = await db.select().from(alertAcks).where(eq(alertAcks.alertKey, alertKey)).limit(1);
return row ? shape(row, now) : null;
}
export interface SetAckInput {
alertId: string;
note: string;
/** A user to hand it to; null for nobody in particular. */
assigneeId: number | null;
/** How long the acknowledgement lasts; null means until the alert is gone. */
hours: number | null;
actor: string;
}
/** Who an alert can be handed to: people who can act on it. */
export async function assignableUsers(): Promise<{ id: number; name: string; role: string }[]> {
const rows = await db.select().from(users).where(inArray(users.role, ["admin", "operator"]));
return rows.map((u) => ({ id: u.id, name: u.name ?? u.email ?? u.oidcSub, role: u.role })).sort((a, b) => a.name.localeCompare(b.name));
}
/** Saves an acknowledgement, replacing an earlier one on the same alert. Returns what changed, for the audit log. */
export async function setAck(input: SetAckInput, now: Date = new Date()): Promise<{ ack: AckInfo; before: AckInfo | null }> {
const note = input.note.trim();
if (note.length > MAX_NOTE) throw new AckError(400, "note_too_long", `Keep the note under ${MAX_NOTE} characters.`);
if (input.hours !== null && !(Number.isInteger(input.hours) && input.hours >= 1 && input.hours <= MAX_HOURS)) throw new AckError(400, "invalid_duration", "Choose how long it lasts, up to thirty days — or until the alert is gone.");
let assigneeName: string | null = null;
if (input.assigneeId !== null) {
const [u] = await db.select().from(users).where(eq(users.id, input.assigneeId)).limit(1);
if (!u || (u.role !== "admin" && u.role !== "operator")) throw new AckError(400, "invalid_assignee", "An alert can be handed to an administrator or an operator.");
assigneeName = u.name ?? u.email ?? u.oidcSub;
}
const before = await loadAck(input.alertId, now);
const until = input.hours === null ? null : new Date(now.getTime() + input.hours * 3600_000).toISOString();
const values = { note, assigneeUserId: input.assigneeId, assigneeName, ackedBy: input.actor, ackedAt: now.toISOString(), until, lastSeenAt: now.toISOString() };
await db.insert(alertAcks).values({ alertKey: input.alertId, ...values }).onConflictDoUpdate({ target: alertAcks.alertKey, set: values });
return { ack: (await loadAck(input.alertId, now))!, before };
}
export async function clearAck(alertId: string): Promise<AckInfo | null> {
const before = await loadAck(alertId);
if (!before) return null;
await db.delete(alertAcks).where(eq(alertAcks.alertKey, alertId));
return before;
}
/**
* Called after every collection of the alerts: the acknowledgements of alerts that are there are marked as seen, and those of alerts that
* have been gone for half an hour are dropped. Returns how many were dropped.
*/
export async function reconcileAcks(presentIds: Set<string>, now: Date = new Date()): Promise<number> {
const rows = await db.select().from(alertAcks);
const seen = rows.filter((r) => presentIds.has(r.alertKey)).map((r) => r.id);
if (seen.length > 0) await db.update(alertAcks).set({ lastSeenAt: now.toISOString() }).where(inArray(alertAcks.id, seen));
const cutoff = new Date(now.getTime() - GONE_AFTER_MS).toISOString();
const gone = rows.filter((r) => !presentIds.has(r.alertKey) && r.lastSeenAt < cutoff).map((r) => r.id);
if (gone.length > 0) await db.delete(alertAcks).where(and(inArray(alertAcks.id, gone), lt(alertAcks.lastSeenAt, cutoff)));
return gone.length;
}
/** What still needs somebody to look: the alerts that aren't silenced and that nobody has acknowledged (or whose acknowledgement has run out). */
export function attentionCounts(alerts: Pick<Alert, "id" | "severity" | "silenced">[], acks: Map<string, AckInfo>): { critical: number; warning: number; acknowledged: number } {
let critical = 0;
let warning = 0;
let acknowledged = 0;
for (const a of alerts) {
if (a.silenced) continue;
if (acks.get(a.id)?.active) {
acknowledged++;
continue;
}
if (a.severity === "critical") critical++;
else if (a.severity === "warning") warning++;
}
return { critical, warning, acknowledged };
}
+29
View File
@@ -15,6 +15,9 @@ import { integrations, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
import { recordFromReport } from "./incidents.js";
import { reconcileAcks } from "./alertAcks.js";
import { lastAttempt as lastSelfBackup, newestSnapshot, STALE_AFTER_HOURS as SELF_BACKUP_STALE_HOURS } from "./selfBackup.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
@@ -53,6 +56,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
gitea: { label: "Gitea", link: "/gitea" },
dockhand: { label: "Docker", link: "/docker" },
tailscale: { label: "Tailscale", link: "/tailscale" },
homelab: { label: "Homelab Manager", link: "/settings/backup" },
pbs: { label: "Proxmox Backup", link: "/pbs" },
uptimekuma: { label: "Uptime Kuma", link: "/uptime-kuma" },
osticket: { label: "osTicket", link: "/osticket" },
@@ -262,6 +266,29 @@ export async function collectAlerts(): Promise<AlertsReport> {
}
}),
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
// stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it.
check("Homelab Manager's own backups", async () => {
const { selfBackup } = await getSettings();
if (!selfBackup.enabled) return;
const newest = newestSnapshot();
const last = await lastSelfBackup();
const failing = last && !last.ok ? ` The last attempt failed: ${last.error}` : "";
if (!newest) {
if (failing) push({ category: "backup", severity: "warning", type: "homelab", message: `Homelab Manager has no backup of its own yet.${failing}`, key: "self-backup" });
return;
}
const ageHours = (Date.now() - Date.parse(newest.at)) / 3600_000;
if (ageHours <= SELF_BACKUP_STALE_HOURS) return;
push({
category: "backup",
severity: ageHours >= 24 * 7 ? "critical" : "warning",
type: "homelab",
message: `The newest backup of Homelab Manager's own database is ${ageHours < 72 ? `${Math.round(ageHours)} h` : `${Math.floor(ageHours / 24)} days`} old.${failing}`,
key: "self-backup",
});
}),
check("Backup verification", async () => {
const { problems, sourceFailures } = await collectPbsProblems({ skipSilenced: false });
for (const p of problems) {
@@ -475,6 +502,8 @@ export async function getAlerts(force: boolean): Promise<AlertsReport & { cached
inFlight ??= collectAlerts()
.then((report) => {
cache = { at: Date.now(), report };
// Remembered for the incident record and the acknowledgements — in the background, and never allowed to hold up or break the answer.
void recordFromReport(report).then(() => reconcileAcks(new Set(report.alerts.map((a) => a.id)))).catch((err) => console.error("[alerts] couldn't update the incident record:", err));
return report;
})
.finally(() => {
+326
View File
@@ -0,0 +1,326 @@
/**
* Incidents: each time an alert is raised and later goes away is one incident, with the moment it began and the moment it ended — so
* "what happened on Tuesday night?" has an answer after the alert itself is long gone. Around each one the page puts together everything the
* app knows about that time: who acknowledged it and why, the maintenance windows, what was changed on the server, the update runs, the
* other alerts that came with it, and the notes people wrote.
*
* The record is made from the alerts as they are collected (and a check every five minutes so it doesn't depend on anybody looking), by
* comparing what is there now with what was open. An alert that isn't there is only called resolved after it has been missing for a couple
* of minutes in a row — much longer when a check couldn't run, since "not there" then may only mean "couldn't look" — and never in the first
* minutes after the app starts, when server alerts are held back on purpose.
*/
import { and, asc, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or } from "drizzle-orm";
import { db } from "../db/client.js";
import { alertIncidents, auditLog, incidentNotes, maintenanceWindows, updateRuns } from "../db/schema.js";
import type { Alert } from "./alertTypes.js";
import { describeAudit, auditTimeToIso, type ActivityTone } from "./serverActivity.js";
import { startupGraceRemainingMs } from "./healthMonitor.js";
import { loadAck, type AckInfo } from "./alertAcks.js";
export type IncidentRow = typeof alertIncidents.$inferSelect;
/** How long an alert must be missing before the incident is called over. */
export const MISSING_GRACE_MS = 2 * 60_000;
/** The same, when a check couldn't run on the last look: not there may then only mean couldn't look. */
export const MISSING_GRACE_BLIND_MS = 30 * 60_000;
/** Resolved incidents are kept this long. */
export const KEEP_DAYS = 400;
/** Other alerts raised this close to an incident, about the same thing, are shown with it. */
export const RELATED_WINDOW_MS = 15 * 60_000;
const RANK: Record<string, number> = { info: 0, warning: 1, critical: 2 };
export interface SnapshotInput {
alerts: Alert[];
/** The checks that couldn't run on this look. */
blind: number;
generatedAt?: string;
}
export interface SnapshotResult {
opened: number[];
resolved: number[];
}
/**
* Brings the incidents in line with the alerts as they are now. Information-only alerts aren't incidents. Safe to call as often as likes:
* the same look twice changes nothing.
*/
export async function recordAlertSnapshot(input: SnapshotInput, now: Date = new Date(), options: { inGrace?: boolean } = {}): Promise<SnapshotResult> {
const iso = now.toISOString();
const present = new Map(input.alerts.filter((a) => a.severity !== "info").map((a) => [a.id, a]));
const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt));
const openByKey = new Map(open.map((i) => [i.alertKey, i]));
const result: SnapshotResult = { opened: [], resolved: [] };
for (const [key, a] of present) {
const existing = openByKey.get(key);
if (existing) {
const peak = (RANK[a.severity] ?? 0) > (RANK[existing.peakSeverity] ?? 0) ? a.severity : existing.peakSeverity;
await db
.update(alertIncidents)
.set({ severity: a.severity, peakSeverity: peak, message: a.message, causedBy: a.causedBy?.name ?? null, lastSeenAt: iso, missingSince: null })
.where(eq(alertIncidents.id, existing.id));
} else {
const [row] = await db
.insert(alertIncidents)
.values({
alertKey: key,
severity: a.severity,
peakSeverity: a.severity,
category: a.category,
source: a.source,
message: a.message,
link: a.link,
targetType: a.target?.type ?? null,
targetId: a.target?.id ?? null,
targetName: a.target?.name ?? null,
causedBy: a.causedBy?.name ?? null,
openedAt: iso,
lastSeenAt: iso,
})
.returning({ id: alertIncidents.id });
result.opened.push(row.id);
}
}
// What was open and isn't there now. Nothing is called over while the app is still settling after a start, when server alerts are held back.
const grace = input.blind > 0 ? MISSING_GRACE_BLIND_MS : MISSING_GRACE_MS;
for (const inc of open) {
if (present.has(inc.alertKey)) continue;
if (options.inGrace) continue;
if (inc.missingSince === null) {
await db.update(alertIncidents).set({ missingSince: iso }).where(eq(alertIncidents.id, inc.id));
} else if (now.getTime() - Date.parse(inc.missingSince) >= grace) {
// Over since the first look it was missing, not since this one.
await db.update(alertIncidents).set({ resolvedAt: inc.missingSince, missingSince: null }).where(eq(alertIncidents.id, inc.id));
result.resolved.push(inc.id);
}
}
return result;
}
/** Called by the collection of the alerts: records what it found, and never lets a problem here stop the alerts from being served. */
export async function recordFromReport(report: { alerts: Alert[]; couldntCheck: unknown[] }, now: Date = new Date()): Promise<void> {
try {
await recordAlertSnapshot({ alerts: report.alerts, blind: report.couldntCheck.length }, now, { inGrace: startupGraceRemainingMs(now.getTime()) > 0 });
} catch (err) {
console.error("[incidents] couldn't record the alerts:", err);
}
}
/** Resolved incidents older than the keeping time are removed (with their notes). */
export async function pruneIncidents(now: Date = new Date()): Promise<number> {
const cutoff = new Date(now.getTime() - KEEP_DAYS * 86_400_000).toISOString();
const old = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(and(isNotNull(alertIncidents.resolvedAt), lte(alertIncidents.resolvedAt, cutoff)));
if (old.length === 0) return 0;
await db.delete(alertIncidents).where(inArray(alertIncidents.id, old.map((o) => o.id)));
return old.length;
}
// ─── Reading them ───────────────────────────────────────────────────────────
export interface IncidentSummary {
id: number;
alertKey: string;
severity: string;
peakSeverity: string;
category: string;
source: string;
message: string;
link: string | null;
target: { type: string; id: number; name: string } | null;
causedBy: string | null;
openedAt: string;
resolvedAt: string | null;
/** In milliseconds: until it was resolved, or until now while it is open. */
durationMs: number;
open: boolean;
notes: number;
}
const summarize = (r: IncidentRow, notes: number, now: Date): IncidentSummary => ({
id: r.id,
alertKey: r.alertKey,
severity: r.severity,
peakSeverity: r.peakSeverity,
category: r.category,
source: r.source,
message: r.message,
link: r.link,
target: r.targetType && r.targetId !== null ? { type: r.targetType, id: r.targetId, name: r.targetName ?? `${r.targetType} #${r.targetId}` } : null,
causedBy: r.causedBy,
openedAt: r.openedAt,
resolvedAt: r.resolvedAt,
durationMs: Math.max(0, (r.resolvedAt ? Date.parse(r.resolvedAt) : now.getTime()) - Date.parse(r.openedAt)),
open: r.resolvedAt === null,
notes,
});
/** Open incidents first (newest first), then the resolved ones, newest first. */
export async function listIncidents(options: { limit?: number; now?: Date } = {}): Promise<IncidentSummary[]> {
const now = options.now ?? new Date();
const limit = options.limit ?? 300;
const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt));
const closed = await db.select().from(alertIncidents).where(isNotNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt)).limit(limit);
const rows = [...open, ...closed];
const noteRows = rows.length > 0 ? await db.select({ incidentId: incidentNotes.incidentId }).from(incidentNotes).where(inArray(incidentNotes.incidentId, rows.map((r) => r.id))) : [];
const counts = new Map<number, number>();
for (const n of noteRows) counts.set(n.incidentId, (counts.get(n.incidentId) ?? 0) + 1);
return rows.map((r) => summarize(r, counts.get(r.id) ?? 0, now));
}
/** The open incident for each alert id, so the Alerts page can link to it. */
export async function openIncidentIds(alertKeys: string[]): Promise<Map<string, number>> {
if (alertKeys.length === 0) return new Map();
const rows = await db.select({ id: alertIncidents.id, key: alertIncidents.alertKey }).from(alertIncidents).where(and(isNull(alertIncidents.resolvedAt), inArray(alertIncidents.alertKey, alertKeys)));
return new Map(rows.map((r) => [r.key, r.id]));
}
export type TimelineKind = "alert" | "ack" | "maintenance" | "change" | "health" | "run" | "related" | "note";
export interface TimelineEvent {
/** ISO. */
at: string;
kind: TimelineKind;
tone: ActivityTone;
title: string;
detail: string | null;
actor: string | null;
/** For a related alert or a note: what it points at. */
incidentId?: number;
noteId?: number;
}
export interface IncidentDetail {
incident: IncidentSummary;
/** The acknowledgement in force now, if the alert is still open. */
ack: AckInfo | null;
timeline: TimelineEvent[];
notes: { id: number; author: string; body: string; at: string }[];
}
function parse(detail: string | null): Record<string, unknown> {
if (!detail) return {};
try {
const v = JSON.parse(detail);
return v && typeof v === "object" ? (v as Record<string, unknown>) : {};
} catch {
return {};
}
}
const words = (ms: number) => {
const m = Math.round(ms / 60_000);
if (m < 90) return `${Math.max(1, m)} min`;
const h = ms / 3600_000;
return h < 48 ? `${Math.round(h * 10) / 10} h` : `${Math.round(h / 24)} days`;
};
/** Everything the app knows about one incident, in the order it happened. */
export async function getIncident(id: number, now: Date = new Date()): Promise<IncidentDetail | null> {
const [row] = await db.select().from(alertIncidents).where(eq(alertIncidents.id, id)).limit(1);
if (!row) return null;
const notes = await db.select().from(incidentNotes).where(eq(incidentNotes.incidentId, id)).orderBy(asc(incidentNotes.createdAt));
const incident = summarize(row, notes.length, now);
const from = Date.parse(row.openedAt);
const to = row.resolvedAt ? Date.parse(row.resolvedAt) : now.getTime();
const fromIso = new Date(from - 30 * 60_000).toISOString();
const toIso = new Date(to + 15 * 60_000).toISOString();
const events: TimelineEvent[] = [];
events.push({ at: row.openedAt, kind: "alert", tone: row.peakSeverity === "critical" ? "bad" : "info", title: `Alert raised (${row.severity})`, detail: row.message, actor: null });
if (row.peakSeverity !== row.severity) {
events.push({ at: row.resolvedAt ?? row.lastSeenAt, kind: "alert", tone: "neutral", title: `At its worst it was ${row.peakSeverity}`, detail: null, actor: null });
}
if (row.causedBy) events.push({ at: row.openedAt, kind: "alert", tone: "info", title: `Held back: caused by ${row.causedBy}`, detail: "Folded under the alert for the machine it runs on.", actor: null });
if (row.resolvedAt) events.push({ at: row.resolvedAt, kind: "alert", tone: "good", title: "Resolved", detail: `It lasted ${words(to - from)}.`, actor: null });
// Acknowledgements and hand-overs of this alert, from the audit log.
const audits = await db.select().from(auditLog).where(and(gte(auditLog.createdAt, fromIso.slice(0, 19).replace("T", " ")), lte(auditLog.createdAt, toIso.slice(0, 19).replace("T", " ")))).orderBy(asc(auditLog.createdAt));
for (const a of audits) {
const d = parse(a.detail);
if (a.category === "alert") {
if (d.alertId !== row.alertKey) continue;
const described = describeAudit(a.category, a.action, d);
events.push({ at: auditTimeToIso(a.createdAt), kind: "ack", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel });
continue;
}
// Things done to, or noticed about, the thing the alert is about.
if (row.targetType && row.targetId !== null && a.targetType === row.targetType && a.targetId === String(row.targetId) && a.category !== "maintenance") {
const described = describeAudit(a.category, a.action, d);
events.push({ at: auditTimeToIso(a.createdAt), kind: described.kind === "health" ? "health" : "change", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel });
}
}
if (row.targetType && row.targetId !== null) {
const windows = await db
.select()
.from(maintenanceWindows)
.where(and(eq(maintenanceWindows.targetType, row.targetType as "server"), eq(maintenanceWindows.targetId, row.targetId), lte(maintenanceWindows.startedAt, new Date(to).toISOString()), gte(maintenanceWindows.endsAt, new Date(from).toISOString())));
for (const w of windows) {
if (w.endsAt <= w.startedAt) continue; // withdrawn before it began
events.push({ at: w.startedAt, kind: "maintenance", tone: "info", title: "Maintenance began — alerts silenced", detail: w.reason, actor: w.createdBy });
if (w.endsAt <= toIso) events.push({ at: w.endsAt, kind: "maintenance", tone: "neutral", title: "Maintenance ended", detail: null, actor: null });
}
if (row.targetType === "server") {
const runs = await db.select().from(updateRuns).where(and(eq(updateRuns.serverId, row.targetId), lte(updateRuns.startedAt, new Date(to).toISOString()), or(isNull(updateRuns.finishedAt), gte(updateRuns.finishedAt, new Date(from).toISOString()))!));
for (const r of runs) {
events.push({ at: r.startedAt, kind: "run", tone: "info", title: `Update run started: ${r.templateName}`, detail: `by ${r.startedBy ?? "someone"}`, actor: r.startedBy });
if (r.finishedAt) events.push({ at: r.finishedAt, kind: "run", tone: r.status === "done" ? "good" : "bad", title: r.status === "done" ? "Update run finished" : r.status === "cancelled" ? "Update run stopped being followed" : "Update run didn't finish", detail: r.message, actor: null });
}
}
// Other alerts about the same thing that began around the same time.
const related = await db
.select()
.from(alertIncidents)
.where(and(eq(alertIncidents.targetType, row.targetType), eq(alertIncidents.targetId, row.targetId), ne(alertIncidents.id, row.id), gte(alertIncidents.openedAt, new Date(from - RELATED_WINDOW_MS).toISOString()), lte(alertIncidents.openedAt, new Date(Math.max(to, from) + RELATED_WINDOW_MS).toISOString())));
for (const r of related) events.push({ at: r.openedAt, kind: "related", tone: "info", title: `Another alert was raised: ${r.category} (${r.severity})`, detail: r.message, actor: null, incidentId: r.id });
}
for (const n of notes) events.push({ at: n.createdAt, kind: "note", tone: "neutral", title: `${n.author} wrote a note`, detail: n.body, actor: n.author, noteId: n.id });
events.sort((a, b) => a.at.localeCompare(b.at));
return { incident, ack: row.resolvedAt ? null : await loadAck(row.alertKey, now), timeline: events, notes: notes.map((n) => ({ id: n.id, author: n.author, body: n.body, at: n.createdAt })) };
}
export const MAX_NOTE = 4000;
export async function addNote(incidentId: number, author: string, body: string, now: Date = new Date()): Promise<{ id: number } | null> {
const [inc] = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(eq(alertIncidents.id, incidentId)).limit(1);
if (!inc) return null;
const [row] = await db.insert(incidentNotes).values({ incidentId, author, body: body.trim(), createdAt: now.toISOString() }).returning({ id: incidentNotes.id });
return row;
}
export async function removeNote(incidentId: number, noteId: number, actor: { name: string; isAdmin: boolean }): Promise<"removed" | "not_found" | "forbidden"> {
const [n] = await db.select().from(incidentNotes).where(and(eq(incidentNotes.id, noteId), eq(incidentNotes.incidentId, incidentId))).limit(1);
if (!n) return "not_found";
if (n.author !== actor.name && !actor.isAdmin) return "forbidden";
await db.delete(incidentNotes).where(eq(incidentNotes.id, noteId));
return "removed";
}
let timer: ReturnType<typeof setInterval> | null = null;
/** A look at the alerts every five minutes, so incidents are recorded whether or not anybody has the page open. Once a day it tidies up old ones. */
export function initIncidentRecorder(lookAtAlerts: () => Promise<unknown>): void {
if (timer) return;
let lastPrune = 0;
const tick = async () => {
try {
await lookAtAlerts();
if (Date.now() - lastPrune > 24 * 3600_000) {
lastPrune = Date.now();
await pruneIncidents();
}
} catch (err) {
console.error("[incidents] the five-minute look failed:", err);
}
};
setTimeout(() => void tick(), 60_000).unref();
timer = setInterval(() => void tick(), 5 * 60_000);
timer.unref();
}
+18
View File
@@ -13,6 +13,9 @@ import { parsePendingUpdates } from "./pendingUpdates.js";
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
import { parseThresholds } from "./serverThresholds.js";
import { getCachedBackupCoverage } from "./backupCoverage.js";
import { lastAttempt as lastSelfBackup, listSnapshots } from "./selfBackup.js";
import { loadAcks, attentionCounts } from "./alertAcks.js";
import { listIncidents } from "./incidents.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { computeSecretStatus } from "./secretStatus.js";
import { getSettings } from "./settingsStore.js";
@@ -115,6 +118,18 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
console.error("[metrics] couldn't read the uptime:", err);
}
// ── Homelab Manager's own backups ──
try {
const snapshots = listSnapshots();
out.add("homelab_selfbackup_snapshots", "gauge", "How many backups of Homelab Manager's own database are kept.", snapshots.length);
const newest = snapshots.find((s) => s.kind === "scheduled" || s.kind === "manual");
if (newest) out.add("homelab_selfbackup_last_success_timestamp_seconds", "gauge", "When the newest backup of Homelab Manager's own database was made, as a Unix time.", Math.floor(Date.parse(newest.at) / 1000));
const last = await lastSelfBackup();
if (last) out.add("homelab_selfbackup_last_attempt_ok", "gauge", "1 if the last attempt to back up Homelab Manager's own database worked, 0 if it failed.", last.ok ? 1 : 0);
} catch (err) {
console.error("[metrics] couldn't read the backups folder:", err);
}
// ── Alerts ──
try {
const report = await getAlerts(false);
@@ -127,6 +142,9 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
}
}
for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity });
// Somebody has said they are dealing with these (still raised, and still counted above).
out.add("homelab_alerts_acknowledged", "gauge", "Alerts somebody has acknowledged and that are not silenced by maintenance.", attentionCounts(report.alerts, await loadAcks(now)).acknowledged);
out.add("homelab_incidents_open", "gauge", "Incidents that have begun and not yet ended.", (await listIncidents({ limit: 1, now })).filter((i) => i.open).length);
out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced);
out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped);
} catch (err) {
+323
View File
@@ -0,0 +1,323 @@
/**
* Backups of Homelab Manager itself. Everything it knows is one SQLite file, so a backup is a consistent copy of that file (`VACUUM INTO`,
* which is safe while the app is working) kept in the data folder's `backups/`: taken every day, the newest few kept, each one checked as
* it is made, and downloadable, uploadable and restorable from Settings → Backup.
*
* What a backup is not: it does not hold `CREDENTIALS_ENCRYPTION_KEY`, which decrypts the credentials stored in it, and it isn't off the
* machine until someone copies it there. Both are said where backups are shown.
*
* A restore can't swap a database that is open, so it is staged (checked, then copied beside the database) and takes effect when the app
* next starts — see db/applyRestore.ts, which does it before anything opens the file.
*/
import { createClient } from "@libsql/client";
import { copyFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, openSync, readdirSync, readFileSync, readSync, closeSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { client, dbPath } from "../db/client.js";
import { PENDING_FILE, PENDING_NOTE, RESULT_NOTE, type RestoreResult } from "../db/restoreFiles.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export type SnapshotKind = "scheduled" | "manual" | "pre-restore" | "uploaded";
export interface SnapshotInfo {
name: string;
kind: SnapshotKind;
sizeBytes: number;
/** ISO — when the file was written. */
at: string;
}
export interface SnapshotCheck {
ok: boolean;
/** SQLite's own verdict ("ok", or what it found). */
integrity: string | null;
/** How many schema changes it has had; goes up with each release that changes the database. */
migrations: number | null;
/** How many this version of the app knows. */
appMigrations: number;
/** Made by a newer version of the app than this one: it can't be restored here. */
newerThanApp: boolean;
counts: { users: number; servers: number; integrations: number } | null;
/** Why it isn't usable, when it isn't. */
problem: string | null;
}
const NAME = /^homelab-manager-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}(?:-(manual|pre-restore|uploaded))?(?:-\d+)?\.sqlite$/;
export const isSnapshotName = (name: string) => NAME.test(name);
const dataDir = () => dirname(dbPath);
export const backupDir = () => join(dataDir(), "backups");
const stamp = (d: Date) => d.toISOString().slice(0, 19).replace(/[:T]/g, "-");
function kindOf(name: string): SnapshotKind {
const m = NAME.exec(name);
return (m?.[1] as SnapshotKind | undefined) ?? "scheduled";
}
/** How many schema changes this version of the app has (the entries in its migration journal). */
export function appMigrationCount(): number {
try {
const journal = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "drizzle", "meta", "_journal.json"), "utf8")) as { entries?: unknown[] };
return journal.entries?.length ?? 0;
} catch {
return 0;
}
}
function hasSqliteHeader(file: string): boolean {
try {
const fd = openSync(file, "r");
try {
const buf = Buffer.alloc(16);
readSync(fd, buf, 0, 16, 0);
return buf.toString("latin1") === "SQLite format 3\0";
} finally {
closeSync(fd);
}
} catch {
return false;
}
}
/** A scratch folder for looking at a copy of a backup, and the way to be rid of it (best effort: some systems hold an opened database until the process ends). */
function scratch(): { file: string; done: () => void } {
const folder = mkdtempSync(join(tmpdir(), "hlm-backup-"));
return {
file: join(folder, "db.sqlite"),
done: () => {
try {
rmSync(folder, { recursive: true, force: true });
} catch {
// left for the system's own tidying of its temporary folder
}
},
};
}
/**
* Looks at a backup: does SQLite think it is sound, is it one of ours, and is it from this version or an older one? It looks at a copy in a
* temporary folder, never at the file itself, so a backup in the backups folder is never held open — it can always be deleted, anywhere.
*/
export async function checkSnapshotFile(file: string): Promise<SnapshotCheck> {
if (!existsSync(file)) return checkOpened(file);
if (!hasSqliteHeader(file)) return checkOpened(file);
const tmp = scratch();
try {
copyFileSync(file, tmp.file);
return await checkOpened(tmp.file);
} finally {
tmp.done();
}
}
/** The check itself, on a file nobody else cares about the handle of. */
async function checkOpened(file: string): Promise<SnapshotCheck> {
const appMigrations = appMigrationCount();
const fail = (problem: string, extra: Partial<SnapshotCheck> = {}): SnapshotCheck => ({ ok: false, integrity: null, migrations: null, appMigrations, newerThanApp: false, counts: null, problem, ...extra });
if (!existsSync(file)) return fail("The file isn't there.");
if (!hasSqliteHeader(file)) return fail("That isn't a database file.");
const c = createClient({ url: `file:${file}` });
try {
const one = async (sql: string) => Object.values((await c.execute(sql)).rows[0] ?? {})[0];
const integrity = String(await one("PRAGMA integrity_check"));
const tables = new Set((await c.execute("select name from sqlite_master where type = 'table'")).rows.map((r) => String(r.name)));
for (const needed of ["users", "servers", "integrations", "settings", "__drizzle_migrations"]) {
if (!tables.has(needed)) return fail(`It isn't a Homelab Manager database (no "${needed}" table).`, { integrity });
}
const migrations = Number(await one("select count(*) from __drizzle_migrations"));
const counts = { users: Number(await one("select count(*) from users")), servers: Number(await one("select count(*) from servers")), integrations: Number(await one("select count(*) from integrations")) };
const newerThanApp = appMigrations > 0 && migrations > appMigrations;
if (integrity !== "ok") return fail(`SQLite found problems in it: ${integrity}`, { integrity, migrations, counts, newerThanApp });
if (newerThanApp) return fail("It was made by a newer version of Homelab Manager than this one, and an older version can't use it. Update first.", { integrity, migrations, counts, newerThanApp });
return { ok: true, integrity, migrations, appMigrations, newerThanApp: false, counts, problem: null };
} catch (err) {
return fail(`It couldn't be read as a database: ${err instanceof Error ? err.message : String(err)}`);
} finally {
c.close();
}
}
/** A name for a new file that isn't taken. */
function freshName(kind: SnapshotKind, now: Date): string {
const base = `homelab-manager-${stamp(now)}${kind === "scheduled" ? "" : `-${kind}`}`;
let name = `${base}.sqlite`;
for (let n = 2; existsSync(join(backupDir(), name)); n++) name = `${base}-${n}.sqlite`;
return name;
}
export interface LastAttempt {
at: string;
ok: boolean;
kind: SnapshotKind;
error: string | null;
name: string | null;
}
const ATTEMPT_FLAG = "selfBackupLastAttempt";
export async function lastAttempt(): Promise<LastAttempt | null> {
try {
const v = await getInternalFlag(ATTEMPT_FLAG);
return v ? (JSON.parse(v) as LastAttempt) : null;
} catch {
return null;
}
}
/** Writes a consistent copy of the database into the backups folder and checks it. Throws, saying why, if it can't — and records the attempt either way. */
export async function createSnapshot(kind: SnapshotKind = "manual", now: Date = new Date()): Promise<{ info: SnapshotInfo; check: SnapshotCheck }> {
const tmp = scratch();
try {
mkdirSync(backupDir(), { recursive: true });
const name = freshName(kind, now);
const file = join(backupDir(), name);
// Made and checked in a temporary folder first, and only then put in the backups folder: what is there is always whole and sound.
await client.execute(`VACUUM INTO '${tmp.file.replace(/'/g, "''")}'`);
const check = await checkOpened(tmp.file);
if (!check.ok) throw new Error(`The copy didn't check out, so it was thrown away: ${check.problem}`);
copyFileSync(tmp.file, file);
try {
chmodSync(file, 0o600);
} catch {
// not every file system has modes
}
const info: SnapshotInfo = { name, kind, sizeBytes: statSync(file).size, at: now.toISOString() };
await setInternalFlag(ATTEMPT_FLAG, JSON.stringify({ at: now.toISOString(), ok: true, kind, error: null, name } satisfies LastAttempt));
return { info, check };
} catch (err) {
const error = err instanceof Error ? err.message : String(err);
await setInternalFlag(ATTEMPT_FLAG, JSON.stringify({ at: now.toISOString(), ok: false, kind, error, name: null } satisfies LastAttempt));
throw err;
} finally {
tmp.done();
}
}
export function listSnapshots(): SnapshotInfo[] {
if (!existsSync(backupDir())) return [];
return readdirSync(backupDir())
.filter(isSnapshotName)
.map((name) => {
const st = statSync(join(backupDir(), name));
return { name, kind: kindOf(name), sizeBytes: st.size, at: st.mtime.toISOString() };
})
.sort((a, b) => b.at.localeCompare(a.at) || b.name.localeCompare(a.name));
}
/** What is kept: the newest `keep` scheduled backups, and a few of each other kind (a restore's "before" copy is worth keeping longer than a day). */
export const KEEP_OTHER: Record<Exclude<SnapshotKind, "scheduled">, number> = { manual: 10, "pre-restore": 3, uploaded: 3 };
export function pruneSnapshots(keep: number): string[] {
const removed: string[] = [];
const all = listSnapshots();
const limitFor = (kind: SnapshotKind) => (kind === "scheduled" ? Math.max(1, keep) : KEEP_OTHER[kind]);
for (const kind of ["scheduled", "manual", "pre-restore", "uploaded"] as const) {
for (const s of all.filter((x) => x.kind === kind).slice(limitFor(kind))) {
rmSync(join(backupDir(), s.name), { force: true });
removed.push(s.name);
}
}
return removed;
}
export function snapshotPath(name: string): string | null {
if (!isSnapshotName(name)) return null;
const file = join(backupDir(), name);
return existsSync(file) ? file : null;
}
export function deleteSnapshot(name: string): boolean {
const file = snapshotPath(name);
if (!file) return false;
rmSync(file, { force: true });
return true;
}
/** A file somebody uploaded, kept as a backup if it checks out (and thrown away if not). */
export async function importUploaded(bytes: Buffer, now: Date = new Date()): Promise<{ info: SnapshotInfo; check: SnapshotCheck }> {
const tmp = scratch();
try {
writeFileSync(tmp.file, bytes);
const check = await checkOpened(tmp.file);
if (!check.ok) throw new Error(check.problem ?? "That file isn't usable.");
mkdirSync(backupDir(), { recursive: true });
const name = freshName("uploaded", now);
const file = join(backupDir(), name);
copyFileSync(tmp.file, file);
try {
chmodSync(file, 0o600);
} catch {
// not every file system has modes
}
return { info: { name, kind: "uploaded", sizeBytes: statSync(file).size, at: now.toISOString() }, check };
} finally {
tmp.done();
}
}
// ─── Restoring ──────────────────────────────────────────────────────────────
export interface StagedRestore {
from: string;
stagedAt: string;
by: string | null;
}
export function stagedRestore(): StagedRestore | null {
try {
if (!existsSync(join(dataDir(), PENDING_FILE))) return null;
const note = JSON.parse(readFileSync(join(dataDir(), PENDING_NOTE), "utf8")) as { from?: string; stagedAt?: string; by?: string | null };
return { from: note.from ?? "a backup", stagedAt: note.stagedAt ?? "", by: note.by ?? null };
} catch {
return existsSync(join(dataDir(), PENDING_FILE)) ? { from: "a backup", stagedAt: "", by: null } : null;
}
}
/** Checks a backup and stages it to replace the database the next time the app starts. Throws, saying why, if it can't be used. */
export async function stageRestore(name: string, by: string | null, now: Date = new Date()): Promise<SnapshotCheck> {
const file = snapshotPath(name);
if (!file) throw new Error("There is no backup with that name.");
const check = await checkSnapshotFile(file);
if (!check.ok) throw new Error(check.problem ?? "That backup isn't usable.");
const pending = join(dataDir(), PENDING_FILE);
const tmp = `${pending}.copying`;
copyFileSync(file, tmp);
renameSync(tmp, pending); // whole, or not there at all: the start-up step never sees half a file
try {
chmodSync(pending, 0o600);
} catch {
// not every file system has modes
}
writeFileSync(join(dataDir(), PENDING_NOTE), JSON.stringify({ from: name, stagedAt: now.toISOString(), by }));
return check;
}
export function cancelStagedRestore(): boolean {
const had = existsSync(join(dataDir(), PENDING_FILE));
rmSync(join(dataDir(), PENDING_FILE), { force: true });
rmSync(join(dataDir(), PENDING_NOTE), { force: true });
return had;
}
/** What the last restore did, once, for the page to say. */
export function lastRestore(): RestoreResult | null {
try {
return JSON.parse(readFileSync(join(dataDir(), RESULT_NOTE), "utf8")) as RestoreResult;
} catch {
return null;
}
}
export function dismissLastRestore(): void {
rmSync(join(dataDir(), RESULT_NOTE), { force: true });
}
// ─── How current the backups are ────────────────────────────────────────────
/** Backups older than this are an alert: a day's slack on top of the daily schedule. */
export const STALE_AFTER_HOURS = 50;
export function newestSnapshot(): SnapshotInfo | null {
return listSnapshots().find((s) => s.kind === "scheduled" || s.kind === "manual") ?? null;
}
@@ -0,0 +1,65 @@
import schedule from "node-schedule";
import { getSettings } from "./settingsStore.js";
import { createSnapshot, lastAttempt, newestSnapshot, pruneSnapshots } from "./selfBackup.js";
import { notify } from "./notify.js";
let currentJob: schedule.Job | null = null;
/** The cron rule for "every day at HH:MM". */
export function cronForBackup(time: string): string {
const [h, m] = time.split(":").map(Number);
return `${Number.isFinite(m) ? m : 30} ${Number.isFinite(h) ? h : 3} * * *`;
}
/** One scheduled backup: take it, then drop the oldest beyond how many are kept. Says so through the channels if it fails. */
export async function runScheduledBackup(now: Date = new Date()): Promise<{ name: string } | null> {
const { selfBackup, notifications } = await getSettings();
try {
const { info } = await createSnapshot("scheduled", now);
pruneSnapshots(selfBackup.keep);
return { name: info.name };
} catch (err) {
console.error("[selfBackup] the scheduled backup failed:", err instanceof Error ? err.message : err);
if (notifications.healthAlerts !== false) {
await notify("Homelab Manager — its own backup failed", `The scheduled backup of Homelab Manager's database didn't work: ${err instanceof Error ? err.message : String(err)}`).catch(() => {});
}
return null;
}
}
/** The next time it will run, or null when it is off. */
export function nextBackupRun(): Date | null {
const next = currentJob?.nextInvocation();
return next ? new Date(next.getTime()) : null;
}
/** (Re)schedules the daily backup per the current settings. Call again after they change. */
export async function scheduleSelfBackup(): Promise<void> {
if (currentJob) {
currentJob.cancel();
currentJob = null;
}
const { selfBackup, notifications } = await getSettings();
if (!selfBackup.enabled) {
console.log("[selfBackup] daily backups are off, nothing scheduled");
return;
}
currentJob = schedule.scheduleJob({ rule: cronForBackup(selfBackup.time), tz: notifications.timezone }, () => {
void runScheduledBackup();
});
console.log(`Daily backup scheduled at ${selfBackup.time} (${notifications.timezone})`);
}
/** Schedules it, and takes one soon if the app has been off through its time (or has never taken one). */
export async function initSelfBackupScheduler(): Promise<void> {
await scheduleSelfBackup();
const { selfBackup } = await getSettings();
if (!selfBackup.enabled) return;
const newest = newestSnapshot();
const overdue = !newest || Date.now() - Date.parse(newest.at) > 26 * 3600_000;
// Only retry a failing one at the next scheduled time, not at every restart.
const last = await lastAttempt();
if (overdue && !(last && !last.ok && Date.now() - Date.parse(last.at) < 6 * 3600_000)) {
setTimeout(() => void runScheduledBackup(), 30_000).unref();
}
}
+11
View File
@@ -72,6 +72,17 @@ export function describeAudit(category: string, action: string, detail: Record<s
return { title: "Admin link removed", tone: "neutral", kind: "change", detail: label };
}
}
if (category === "alert") {
const note = typeof detail.note === "string" && detail.note ? detail.note : null;
const assignee = typeof detail.assignee === "string" ? detail.assignee : null;
const lasts = typeof detail.hours === "number" ? `for ${detail.hours >= 48 ? `${Math.round(detail.hours / 24)} days` : `${detail.hours} h`}` : "until the alert is gone";
switch (action) {
case "ack":
return { title: assignee ? `Acknowledged, handed to ${assignee}` : "Acknowledged", tone: "info", kind: "change", detail: [note, lasts].filter(Boolean).join(" · ") };
case "unack":
return { title: "Acknowledgement taken back", tone: "neutral", kind: "change", detail: null };
}
}
if (category === "update_run") {
const template = typeof detail.template === "string" ? detail.template : null;
switch (action) {
+11
View File
@@ -129,6 +129,15 @@ export interface AgentSettings {
autoUpdate: boolean;
}
/** The daily backup of Homelab Manager's own database (see services/selfBackup.ts). On unless an administrator turns it off. */
export interface SelfBackupSettings {
enabled: boolean;
/** "HH:MM", in the notification time zone. */
time: string;
/** How many of the daily backups are kept; the oldest beyond that are deleted. */
keep: number;
}
/** What the inventory needs to work out a running cost. Without a price per kWh it shows draw but no cost. */
export interface InventorySettings {
/** A short name shown after amounts ("SEK", "kr", "$"); may be blank. */
@@ -190,6 +199,7 @@ export interface AppSettings {
digest: DigestSettings;
signIn: SignInSettings;
inventory: InventorySettings;
selfBackup: SelfBackupSettings;
agents: AgentSettings;
consistency: ConsistencySettings;
nameGenerator: NameGeneratorSettings;
@@ -227,6 +237,7 @@ const DEFAULTS: AppSettings = {
digest: { enabled: false, day: 1, time: "08:00" },
inventory: { currency: "", pricePerKwh: null },
agents: { autoUpdate: false },
selfBackup: { enabled: true, time: "03:30", keep: 14 },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
+21
View File
@@ -20,6 +20,8 @@ import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
import { attentionCounts, loadAcks } from "./alertAcks.js";
import { listIncidents } from "./incidents.js";
export const DIGEST_DAYS = 7;
const DAY_MS = 24 * 3600_000;
@@ -90,11 +92,30 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
if (report.counts.silenced > 0) held.push(`${report.counts.silenced} silenced by maintenance`);
if (report.counts.grouped > 0) held.push(`${report.counts.grouped} folded under a server that is down`);
if (held.length > 0) lines.push(`(${held.join(", ")})`);
const acknowledged = attentionCounts(report.alerts, await loadAcks(now)).acknowledged;
if (acknowledged > 0) lines.push(`(${acknowledged} of them acknowledged — somebody is on it)`);
sections.push({ heading: "Right now", lines, attention: active.length > 0 || report.couldntCheck.length > 0 });
} catch (err) {
console.error("[digest] couldn't read the alerts:", err);
}
// ── Incidents ──
try {
const since = now.getTime() - DIGEST_DAYS * DAY_MS;
const all = await listIncidents({ now });
const begun = all.filter((i) => Date.parse(i.openedAt) >= since);
const ended = all.filter((i) => i.resolvedAt !== null && Date.parse(i.resolvedAt) >= since);
const open = all.filter((i) => i.open);
if (begun.length > 0 || ended.length > 0) {
const lines = [`${plural(begun.length, "incident")} began, ${ended.length} ended, ${open.length} still open.`];
const longest = [...ended].sort((a, b) => b.durationMs - a.durationMs).slice(0, 3);
lines.push(...longest.map((i) => `- ${i.message} — lasted ${spanWords(i.durationMs / 60_000)}`));
sections.push({ heading: "Incidents", lines, attention: open.length > 0 });
}
} catch (err) {
console.error("[digest] couldn't read the incidents:", err);
}
// ── Uptime ──
try {
const uptime = await getUptime(now, DIGEST_DAYS);
+453
View File
@@ -0,0 +1,453 @@
/**
* Acknowledging and handing over alerts, and the incident record: when an alert began and ended, and what happened around it.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-incidents-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, auditLog, maintenanceWindows, updateRuns, alertAcks, alertIncidents, incidentNotes } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const acks = await import("../src/services/alertAcks.js");
const inc = await import("../src/services/incidents.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { getServerActivity } = await import("../src/services/serverActivity.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { buildDigest } = await import("../src/services/weeklyDigest.js");
const { alertsRouter } = await import("../src/routes/alerts.js");
const { incidentsRouter } = await import("../src/routes/incidents.js");
const { hashToken } = await import("../src/services/tokens.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const T0 = new Date("2026-10-17T12:00:00.000Z");
const at = (minutes: number) => new Date(T0.getTime() + minutes * 60_000);
const alert = (id: string, extra: Record<string, unknown> = {}) =>
({ id, severity: "warning" as const, category: "disk" as const, source: "Server", message: `${id} is a problem`, link: null, silenced: false, target: null, ...extra }) as never;
let adminId = 0;
let operatorId = 0;
let viewerId = 0;
before(async () => {
const rows = await db
.insert(users)
.values([
{ oidcSub: "ada", name: "Ada", role: "admin" },
{ oidcSub: "olle", name: "Olle", role: "operator" },
{ oidcSub: "vera", name: "Vera", role: "viewer" },
])
.returning();
[adminId, operatorId, viewerId] = rows.map((r) => r.id);
});
after(() => {
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
describe("acknowledging an alert", () => {
test("a note, who has it and for how long are kept; a second one replaces the first; taking it back removes it", async () => {
const first = await acks.setAck({ alertId: "disk:server:1:/", note: " looking at it ", assigneeId: operatorId, hours: 4, actor: "Ada" }, T0);
assert.equal(first.before, null);
assert.equal(first.ack.note, "looking at it");
assert.deepEqual(first.ack.assignee, { id: operatorId, name: "Olle" });
assert.equal(first.ack.ackedBy, "Ada");
assert.equal(first.ack.until, at(240).toISOString());
assert.equal(first.ack.active, true);
const second = await acks.setAck({ alertId: "disk:server:1:/", note: "", assigneeId: null, hours: null, actor: "Olle" }, at(10));
assert.equal(second.before!.note, "looking at it");
assert.equal(second.ack.assignee, null);
assert.equal(second.ack.until, null, "until the alert is gone");
assert.equal((await db.select().from(alertAcks)).length, 1, "one per alert");
assert.equal((await acks.clearAck("disk:server:1:/"))!.ackedBy, "Olle");
assert.equal(await acks.clearAck("disk:server:1:/"), null);
assert.equal((await acks.loadAcks()).size, 0);
});
test("only people who can act can be handed an alert, and the note and the time are limited", async () => {
const refused = async (input: Partial<Parameters<typeof acks.setAck>[0]>, code: string) =>
assert.rejects(acks.setAck({ alertId: "x", note: "", assigneeId: null, hours: null, actor: "Ada", ...input }, T0), (e: unknown) => e instanceof acks.AckError && e.code === code, code);
await refused({ assigneeId: viewerId }, "invalid_assignee");
await refused({ assigneeId: 99999 }, "invalid_assignee");
await refused({ note: "x".repeat(acks.MAX_NOTE + 1) }, "note_too_long");
await refused({ hours: 0 }, "invalid_duration");
await refused({ hours: acks.MAX_HOURS + 1 }, "invalid_duration");
await refused({ hours: 1.5 }, "invalid_duration");
assert.deepEqual((await acks.assignableUsers()).map((u) => u.name), ["Ada", "Olle"], "no viewers in the list");
});
test("it runs out when it was meant to, and only then does the alert count as unattended again", async () => {
await acks.setAck({ alertId: "a:1", note: "", assigneeId: null, hours: 2, actor: "Ada" }, T0);
await acks.setAck({ alertId: "a:2", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
const alerts = [
{ id: "a:1", severity: "critical", silenced: false },
{ id: "a:2", severity: "warning", silenced: false },
{ id: "a:3", severity: "critical", silenced: false },
{ id: "a:4", severity: "warning", silenced: true },
] as { id: string; severity: "critical" | "warning" | "info"; silenced: boolean }[];
assert.deepEqual(acks.attentionCounts(alerts, await acks.loadAcks(at(60))), { critical: 1, warning: 0, acknowledged: 2 }, "a:3 is the only one nobody has looked at; a silenced one is neither");
const later = await acks.loadAcks(at(150));
assert.equal(later.get("a:1")!.active, false);
assert.deepEqual(acks.attentionCounts(alerts, later), { critical: 2, warning: 0, acknowledged: 1 }, "a:1 is unattended again; a:2 has no end");
await db.delete(alertAcks);
});
test("it goes when the alert has been gone for half an hour — not for a blip, and not if the alert is back", async () => {
await acks.setAck({ alertId: "gone", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
await acks.setAck({ alertId: "stays", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(10)), 0, "ten minutes missing is a blip");
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(29)), 0);
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(31)), 1);
assert.deepEqual([...(await acks.loadAcks()).keys()], ["stays"]);
// a problem that returns later is a new one, nobody having looked at it
assert.equal((await acks.loadAcks()).has("gone"), false);
await db.delete(alertAcks);
});
});
describe("the incident record", () => {
const look = (alerts: unknown[], minutes: number, options: { blind?: number; inGrace?: boolean } = {}) => inc.recordAlertSnapshot({ alerts: alerts as never, blind: options.blind ?? 0 }, at(minutes), { inGrace: options.inGrace });
const rows = () => db.select().from(alertIncidents).orderBy(alertIncidents.id);
test("an alert that appears begins an incident; the same look again changes nothing; information-only alerts aren't incidents", async () => {
await db.delete(alertIncidents);
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
const r1 = await look([a, alert("info:thing", { severity: "info" })], 0);
assert.equal(r1.opened.length, 1);
assert.deepEqual((await look([a], 1)).opened, []);
const [row] = await rows();
assert.equal(row.alertKey, "disk:server:1:/");
assert.equal(row.openedAt, at(0).toISOString());
assert.equal(row.resolvedAt, null);
assert.equal(row.targetName, "elsa");
assert.equal(row.lastSeenAt, at(1).toISOString());
assert.equal((await rows()).length, 1);
});
test("it keeps the worst it got to and the latest wording", async () => {
await look([alert("disk:server:1:/", { severity: "critical", message: "now 97% full", target: { type: "server", id: 1, name: "elsa" } })], 5);
await look([alert("disk:server:1:/", { severity: "warning", message: "back to 91% full", target: { type: "server", id: 1, name: "elsa" } })], 6);
const [row] = await rows();
assert.equal(row.severity, "warning");
assert.equal(row.peakSeverity, "critical");
assert.equal(row.message, "back to 91% full");
});
test("it is only called over after being missing for a couple of minutes in a row, and then it ended when it was first missing", async () => {
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
await look([], 10);
let [row] = await rows();
assert.equal(row.resolvedAt, null);
assert.equal(row.missingSince, at(10).toISOString());
await look([a], 11); // it was only a blip
[row] = await rows();
assert.equal(row.missingSince, null, "back before it was called over: the count starts again");
assert.equal(row.resolvedAt, null);
await look([], 20);
await look([], 21);
[row] = await rows();
assert.equal(row.resolvedAt, null, "one minute isn't enough");
const done = await look([], 23);
assert.equal(done.resolved.length, 1);
[row] = await rows();
assert.equal(row.resolvedAt, at(20).toISOString(), "it ended when it first wasn't there, not when that was confirmed");
assert.equal(row.missingSince, null);
});
test("when it comes back it is a new incident, and the old one stays as it was", async () => {
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
const r = await look([a], 60);
assert.equal(r.opened.length, 1);
const all = await rows();
assert.equal(all.length, 2);
assert.equal(all[0].resolvedAt, at(20).toISOString());
assert.equal(all[1].resolvedAt, null);
assert.equal(all[1].peakSeverity, "warning");
});
test("after a check couldn't run it takes much longer to call anything over, and never in the first minutes after the app starts", async () => {
await db.delete(alertIncidents);
const a = alert("proxmox-backup:1:pve1", { category: "backup" });
await look([a], 0);
await look([], 5, { blind: 1 });
await look([], 20, { blind: 1 });
assert.equal((await rows())[0].resolvedAt, null, "fifteen minutes missing isn't enough while a check is failing");
assert.equal((await look([], 36, { blind: 1 })).resolved.length, 1);
await db.delete(alertIncidents);
await look([a], 100);
await look([], 101, { inGrace: true });
await look([], 200, { inGrace: true });
const [row] = await rows();
assert.equal(row.resolvedAt, null, "server alerts are held back on purpose for a while after the app starts, so missing then means nothing");
assert.equal(row.missingSince, null);
});
test("it is recorded from the alerts as they are collected, without anyone asking, and the Alerts page links to it", async () => {
await db.delete(alertIncidents);
const [s] = await db.insert(servers).values({ name: "quiet", apiTokenHash: hashToken("q"), apiTokenPrefix: "q", lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString() }).returning();
invalidateAlertsCache();
await getAlerts(true);
let found: typeof alertIncidents.$inferSelect | undefined;
for (let i = 0; i < 40 && !found; i++) {
found = (await rows()).find((r) => r.alertKey === `offline:offline:server:${s.id}`);
if (!found) await new Promise((r) => setTimeout(r, 50));
}
assert.ok(found, "an incident for the offline server");
assert.equal(found.severity, "critical");
assert.deepEqual([found.targetType, found.targetId, found.targetName], ["server", s.id, "quiet"]);
assert.equal((await inc.openIncidentIds([found.alertKey])).get(found.alertKey), found.id);
await db.delete(servers).where(eq(servers.id, s.id));
});
test("resolved ones are kept for a long while and then removed with their notes; open ones never", async () => {
await db.delete(alertIncidents);
const old = new Date(Date.now() - (inc.KEEP_DAYS + 5) * 86_400_000).toISOString();
const [oldRow] = await db.insert(alertIncidents).values({ alertKey: "old", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "old", openedAt: old, resolvedAt: old, lastSeenAt: old }).returning();
await db.insert(incidentNotes).values({ incidentId: oldRow.id, author: "Ada", body: "n", createdAt: old });
const [openRow] = await db.insert(alertIncidents).values({ alertKey: "ancient-open", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "still open", openedAt: old, lastSeenAt: old }).returning();
assert.equal(await inc.pruneIncidents(), 1);
assert.deepEqual((await rows()).map((r) => r.id), [openRow.id]);
assert.equal((await db.select().from(incidentNotes)).length, 0);
});
});
describe("the timeline around an incident", () => {
let serverId = 0;
let incidentId = 0;
before(async () => {
await db.delete(alertIncidents);
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("e"), apiTokenPrefix: "e" }).returning();
serverId = s.id;
const [row] = await db
.insert(alertIncidents)
.values({ alertKey: `offline:offline:server:${serverId}`, severity: "critical", peakSeverity: "critical", category: "offline", source: "Server", message: "elsa hasn't reported for 2 h", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(0).toISOString(), resolvedAt: at(95).toISOString(), lastSeenAt: at(95).toISOString() })
.returning();
incidentId = row.id;
const stamp = (m: number) => at(m).toISOString().slice(0, 19).replace("T", " ");
const audit = (m: number, category: string, action: string, detail: object, actor = "system", target: object = { targetType: "server", targetId: String(serverId) }) =>
db.insert(auditLog).values({ actorLabel: actor, category, action, detail: JSON.stringify(detail), createdAt: stamp(m), ...target });
await audit(1, "server", "went_offline", { message: "elsa hasn't reported for 2 h" });
await audit(5, "alert", "ack", { alertId: row.alertKey, message: "x", note: "power cut in the cellar", assignee: "Olle", hours: 4 }, "Ada");
await audit(6, "alert", "ack", { alertId: "some:other:alert", note: "not this one" }, "Ada");
await audit(30, "server", "update", { name: "elsa" }, "Ada");
await audit(90, "server", "back_online", {});
await audit(40, "server", "update", {}, "Ada", { targetType: "server", targetId: "9999" }); // another server
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "UPS swap", startedAt: at(20).toISOString(), endsAt: at(60).toISOString(), createdBy: "Ada" });
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "withdrawn", startedAt: at(25).toISOString(), endsAt: at(25).toISOString() });
await db.insert(updateRuns).values({ serverId, integrationId: 1, projectId: 1, templateId: 1, templateName: "Apply updates", hostLimit: "elsa", status: "done", message: "No updates are waiting now; no restart is needed.", startedBy: "Ada", startedAt: at(61).toISOString(), finishedAt: at(80).toISOString() });
await db.insert(alertIncidents).values({ alertKey: `disk:server:${serverId}:/`, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "/ is 93% full", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(8).toISOString(), lastSeenAt: at(8).toISOString() });
await db.insert(alertIncidents).values({ alertKey: `disk:server:${serverId}:/old`, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "long ago", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(-600).toISOString(), lastSeenAt: at(-600).toISOString() });
await inc.addNote(incidentId, "Olle", "Confirmed: the building lost power.", at(15));
});
test("it puts everything in the order it happened, and only what belongs to this incident", async () => {
const d = (await inc.getIncident(incidentId, at(100)))!;
assert.equal(d.incident.open, false);
assert.equal(d.incident.durationMs, 95 * 60_000);
const titles = d.timeline.map((e) => `${e.at.slice(11, 16)} ${e.kind}: ${e.title}`);
assert.deepEqual(titles, [
"12:00 alert: Alert raised (critical)",
"12:01 health: Stopped reporting — marked offline",
"12:05 ack: Acknowledged, handed to Olle",
"12:08 related: Another alert was raised: disk (warning)",
"12:15 note: Olle wrote a note",
"12:20 maintenance: Maintenance began — alerts silenced",
"12:30 change: Details changed",
"12:55 maintenance: Maintenance ended".replace("12:55", "13:00"),
"13:01 run: Update run started: Apply updates",
"13:20 run: Update run finished",
"13:30 health: Reporting again",
"13:35 alert: Resolved",
]);
});
test("the details say what was written and by whom", async () => {
const d = (await inc.getIncident(incidentId, at(100)))!;
const ack = d.timeline.find((e) => e.kind === "ack")!;
assert.equal(ack.actor, "Ada");
assert.match(ack.detail!, /power cut in the cellar · for 4 h/);
const note = d.timeline.find((e) => e.kind === "note")!;
assert.equal(note.detail, "Confirmed: the building lost power.");
assert.ok(note.noteId);
const related = d.timeline.find((e) => e.kind === "related")!;
assert.match(related.detail!, /93% full/);
assert.ok(related.incidentId);
assert.equal(d.timeline.find((e) => e.title === "Resolved")!.detail, "It lasted 1.6 h.");
assert.equal(d.ack, null, "no acknowledgement to show on one that is over");
assert.equal(d.notes.length, 1);
assert.equal((await inc.getIncident(999999)), null);
});
test("an open one runs to now and shows the acknowledgement in force", async () => {
const [row] = await db.select().from(alertIncidents).where(eq(alertIncidents.alertKey, `disk:server:${serverId}:/`));
await acks.setAck({ alertId: row.alertKey, note: "freeing space", assigneeId: operatorId, hours: null, actor: "Ada" }, at(9));
const d = (await inc.getIncident(row.id, at(120)))!;
assert.equal(d.incident.open, true);
assert.equal(d.incident.durationMs, 112 * 60_000);
assert.equal(d.ack!.note, "freeing space");
assert.equal(d.ack!.assignee!.name, "Olle");
assert.ok(!d.timeline.some((e) => e.title === "Resolved"));
await db.delete(alertAcks);
});
test("the list has the open ones first, then the rest newest first, with how many notes each has", async () => {
const list = await inc.listIncidents({ now: at(120) });
assert.equal(list[0].open, true);
const firstClosed = list.findIndex((i) => !i.open);
assert.ok(list.slice(0, firstClosed).every((i) => i.open) && list.slice(firstClosed).every((i) => !i.open));
assert.equal(list.find((i) => i.id === incidentId)!.notes, 1);
assert.deepEqual(list.filter((i) => !i.open).map((i) => i.openedAt), list.filter((i) => !i.open).map((i) => i.openedAt).sort().reverse());
});
});
describe("in the weekly digest", () => {
test("the week's incidents are counted, the longest are named, and acknowledged alerts are said to be", async () => {
const digest = await buildDigest(at(120));
assert.match(digest.message, /INCIDENTS/);
assert.match(digest.message, /\d+ incidents? began, 1 ended, \d+ still open/);
assert.match(digest.message, /elsa hasn't reported for 2 h — lasted 1 h 35 min/);
});
});
describe("through the routes", () => {
let app: Server;
let base = "";
let who = "ada";
let serverId = 0;
let alertId = "";
let incidentId = 0;
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
before(async () => {
await db.delete(alertIncidents);
const [s] = await db.insert(servers).values({ name: "zoe", apiTokenHash: hashToken("z"), apiTokenPrefix: "z", lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString() }).returning();
serverId = s.id;
alertId = `offline:offline:server:${serverId}`.replace("offline:offline", "offline:offline");
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/alerts", alertsRouter);
a.use("/api/incidents", incidentsRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
invalidateAlertsCache();
const report = await (await call("GET", "/alerts?refresh=1")).json();
alertId = report.alerts.find((x: { id: string }) => x.id.includes(`:server:${serverId}`)).id;
for (let i = 0; i < 40 && !incidentId; i++) {
incidentId = (await db.select().from(alertIncidents)).find((r) => r.alertKey === alertId)?.id ?? 0;
if (!incidentId) await new Promise((r) => setTimeout(r, 50));
}
});
after(() => app.close());
test("operators acknowledge and hand over; viewers can look but not; an alert that isn't there can't be", async () => {
who = "vera";
assert.equal((await call("PUT", "/alerts/ack", { alertId })).status, 403);
assert.equal((await call("GET", "/alerts/assignees")).status, 403);
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 403);
assert.equal((await call("GET", "/alerts")).status, 200);
who = "olle";
const names = (await (await call("GET", "/alerts/assignees")).json()).users.map((u: { name: string }) => u.name);
assert.deepEqual(names, ["Ada", "Olle"]);
assert.equal((await call("PUT", "/alerts/ack", { alertId: "no:such:alert" })).status, 404);
assert.equal((await call("PUT", "/alerts/ack", { alertId: "" })).status, 400);
assert.equal((await call("PUT", "/alerts/ack", { alertId, hours: 0 })).status, 400);
const bad = await call("PUT", "/alerts/ack", { alertId, assigneeId: viewerId });
assert.equal(bad.status, 400);
assert.equal((await bad.json()).error, "invalid_assignee");
const res = await call("PUT", "/alerts/ack", { alertId, note: "power cut", assigneeId: adminId, hours: 6 });
assert.equal(res.status, 200);
const { ack } = await res.json();
assert.equal(ack.assignee.name, "Ada");
assert.equal(ack.ackedBy, "Olle");
assert.equal(ack.active, true);
});
test("the Alerts page shows it on the alert, leaves it out of what needs attention, and links to the incident", async () => {
invalidateAlertsCache();
const report = await (await call("GET", "/alerts")).json();
const a = report.alerts.find((x: { id: string }) => x.id === alertId);
assert.equal(a.ack.note, "power cut");
assert.equal(a.incidentId, incidentId);
assert.ok(report.counts.acknowledged >= 1);
assert.ok(report.counts.attention.critical < report.counts.critical, "the whole count is still the whole count; attention leaves out the acknowledged");
const metrics = await buildMetrics();
assert.match(metrics, /homelab_alerts_acknowledged [1-9]/);
assert.match(metrics, /homelab_incidents_open [1-9]/);
});
test("it is in the audit log and in the server's history, and taking it back is too", async () => {
const entry = (await db.select().from(auditLog)).filter((e) => e.category === "alert" && e.action === "ack").pop()!;
assert.equal(entry.targetType, "server");
assert.equal(entry.targetId, String(serverId));
assert.equal(JSON.parse(entry.detail!).assignee, "Ada");
const activity = await getServerActivity(serverId);
assert.ok(activity.some((e) => e.title === "Acknowledged, handed to Ada" && /power cut/.test(e.detail ?? "")));
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 204);
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 404);
invalidateAlertsCache();
assert.equal((await (await call("GET", "/alerts")).json()).alerts.find((x: { id: string }) => x.id === alertId).ack, null);
assert.ok((await getServerActivity(serverId)).some((e) => e.title === "Acknowledgement taken back"));
});
test("everyone signed in can read the incidents; operators write notes; a note can be removed by its author or an administrator", async () => {
who = "vera";
const list = await (await call("GET", "/incidents")).json();
assert.ok(list.incidents.some((i: { id: number }) => i.id === incidentId));
assert.equal((await call("GET", `/incidents/${incidentId}`)).status, 200);
assert.equal((await call("GET", "/incidents/999999")).status, 404);
assert.equal((await call("GET", "/incidents/x")).status, 400);
assert.equal((await call("POST", `/incidents/${incidentId}/notes`, { body: "hello" })).status, 403);
who = "olle";
assert.equal((await call("POST", `/incidents/${incidentId}/notes`, { body: " " })).status, 400);
assert.equal((await call("POST", "/incidents/999999/notes", { body: "x" })).status, 404);
const made = await call("POST", `/incidents/${incidentId}/notes`, { body: "It was the UPS." });
assert.equal(made.status, 201);
const noteId = (await made.json()).id;
const detail = await (await call("GET", `/incidents/${incidentId}`)).json();
assert.ok(detail.timeline.some((e: { kind: string; detail: string }) => e.kind === "note" && e.detail === "It was the UPS."));
// another operator can't remove it, its author and an administrator can
await db.insert(users).values({ oidcSub: "olga", name: "Olga", role: "operator" });
who = "olga";
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 403);
who = "olle";
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 204);
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 404);
const again = await (await call("POST", `/incidents/${incidentId}/notes`, { body: "second" })).json();
who = "ada";
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${again.id}`)).status, 204);
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "incident").map((e) => e.action);
assert.ok(actions.includes("note") && actions.includes("note_removed"));
});
});
+453
View File
@@ -0,0 +1,453 @@
/**
* Backups of Homelab Manager itself: taking them, checking them, keeping the newest, staging and applying a restore, the routes around
* them, and what the alerts and /metrics say about how current they are.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-selfbackup-"));
process.env.DATABASE_PATH = join(dir, "data", "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { createClient } = await import("@libsql/client");
const { db, client } = await import("../src/db/client.js");
const { users, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const sb = await import("../src/services/selfBackup.js");
const { runScheduledBackup, cronForBackup } = await import("../src/services/selfBackupScheduler.js");
const { applyPendingRestore } = await import("../src/db/applyRestore.js");
const { PENDING_FILE, PENDING_NOTE, RESULT_NOTE } = await import("../src/db/restoreFiles.js");
const { dbBackupsRouter } = await import("../src/routes/dbBackups.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { getSettings, updateSettings, setInternalFlag } = await import("../src/services/settingsStore.js");
const schedule = (await import("node-schedule")).default;
await runMigrations();
after(async () => {
await schedule.gracefulShutdown();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const at = (iso: string) => new Date(iso);
/** Opens a copy of a backup, never the backup itself: an opened database can't be deleted on some systems until the process ends. */
const open = (file: string) => {
const copy = join(mkdtempSync(join(dir, "copy-")), "db.sqlite");
copyFileSync(file, copy);
return createClient({ url: `file:${copy}` });
};
const wipeBackups = () => rmSync(sb.backupDir(), { recursive: true, force: true });
describe("taking and checking a backup", () => {
test("a backup is a consistent copy that opens, passes SQLite's check, and holds what was there", async () => {
await db.insert(users).values({ oidcSub: "ada", name: "Ada", role: "admin" });
const { info, check } = await sb.createSnapshot("manual", at("2026-10-17T12:00:00Z"));
assert.equal(info.name, "homelab-manager-2026-10-17-12-00-00-manual.sqlite");
assert.equal(info.kind, "manual");
assert.ok(info.sizeBytes > 10_000);
assert.equal(check.ok, true);
assert.equal(check.integrity, "ok");
assert.equal(check.migrations, sb.appMigrationCount());
assert.equal(check.counts!.users, 1);
// and it really holds it: open the file on its own and read the row back
const copy = open(join(sb.backupDir(), info.name));
const rows = (await copy.execute("select name from users")).rows;
copy.close();
assert.deepEqual(rows.map((r) => r.name), ["Ada"]);
});
test("changes made after it was taken are not in it", async () => {
const { info } = await sb.createSnapshot("manual", at("2026-10-17T12:01:00Z"));
await db.insert(users).values({ oidcSub: "bo", name: "Bo", role: "viewer" });
const copy = open(join(sb.backupDir(), info.name));
assert.equal(Number((await copy.execute("select count(*) as n from users")).rows[0].n), 1);
copy.close();
await db.delete(users).where((await import("drizzle-orm")).eq(users.oidcSub, "bo"));
});
test("two taken in the same second don't collide", async () => {
const a = await sb.createSnapshot("manual", at("2026-10-17T13:00:00Z"));
const b = await sb.createSnapshot("manual", at("2026-10-17T13:00:00Z"));
assert.notEqual(a.info.name, b.info.name);
assert.match(b.info.name, /-manual-2\.sqlite$/);
});
test("files that aren't a usable backup are said to be, and why", async () => {
const junk = join(dir, "junk.sqlite");
writeFileSync(junk, "this is not a database at all");
assert.match((await sb.checkSnapshotFile(junk)).problem!, /isn't a database file/);
assert.match((await sb.checkSnapshotFile(join(dir, "missing.sqlite"))).problem!, /isn't there/);
// a real SQLite file that isn't ours
const other = join(dir, "other.sqlite");
const c = createClient({ url: `file:${other}` });
await c.execute("create table cats (name text)");
c.close();
assert.match((await sb.checkSnapshotFile(other)).problem!, /isn't a Homelab Manager database/);
// one of ours, made by a newer version (more schema changes than this one has)
const newer = join(dir, "newer.sqlite");
await client.execute(`VACUUM INTO '${newer.replace(/'/g, "''")}'`);
const n = createClient({ url: `file:${newer}` });
for (let i = 0; i < 3; i++) await n.execute({ sql: "insert into __drizzle_migrations (hash, created_at) values (?, ?)", args: [`future-${i}`, 9_999_999_999_999 + i] });
n.close();
const check = await sb.checkSnapshotFile(newer);
assert.equal(check.ok, false);
assert.equal(check.newerThanApp, true);
assert.match(check.problem!, /newer version/);
assert.ok(check.migrations! > check.appMigrations);
});
test("an older one is fine: this version brings it up to date when it starts", async () => {
const older = join(dir, "older.sqlite");
await client.execute(`VACUUM INTO '${older.replace(/'/g, "''")}'`);
const o = createClient({ url: `file:${older}` });
await o.execute("delete from __drizzle_migrations where rowid = (select max(rowid) from __drizzle_migrations)");
o.close();
const check = await sb.checkSnapshotFile(older);
assert.equal(check.ok, true);
assert.equal(check.migrations, sb.appMigrationCount() - 1);
});
test("only names of our own files are ever acted on, whatever is asked for", () => {
for (const bad of ["../test.sqlite", "..\\test.sqlite", "homelab-manager-2026-10-17-12-00-00.sqlite/../../x", "homelab-manager-x.sqlite", "test.sqlite", "homelab-manager-2026-10-17-12-00-00.sqlite.bak", "/etc/passwd", ""]) {
assert.equal(sb.isSnapshotName(bad), false, bad);
assert.equal(sb.snapshotPath(bad), null, bad);
assert.equal(sb.deleteSnapshot(bad), false, bad);
}
for (const good of ["homelab-manager-2026-10-17-12-00-00.sqlite", "homelab-manager-2026-10-17-12-00-00-manual.sqlite", "homelab-manager-2026-10-17-12-00-00-pre-restore-2.sqlite", "homelab-manager-2026-10-17-12-00-00-uploaded.sqlite"]) {
assert.equal(sb.isSnapshotName(good), true, good);
}
});
});
describe("keeping the newest", () => {
const fake = (name: string, mtime: string) => {
mkdirSync(sb.backupDir(), { recursive: true });
const f = join(sb.backupDir(), name);
writeFileSync(f, "x");
utimesSync(f, at(mtime), at(mtime));
};
const names = () => sb.listSnapshots().map((s) => s.name);
test("the newest N of the daily ones stay, and each other kind has its own, smaller allowance", () => {
wipeBackups();
for (let d = 1; d <= 6; d++) fake(`homelab-manager-2026-10-0${d}-03-30-00.sqlite`, `2026-10-0${d}T03:30:00Z`);
for (let d = 1; d <= 12; d++) fake(`homelab-manager-2026-10-${String(d).padStart(2, "0")}-09-00-00-manual.sqlite`, `2026-10-${String(d).padStart(2, "0")}T09:00:00Z`);
for (let d = 1; d <= 5; d++) fake(`homelab-manager-2026-10-0${d}-10-00-00-pre-restore.sqlite`, `2026-10-0${d}T10:00:00Z`);
const removed = sb.pruneSnapshots(3);
const left = names();
assert.equal(left.filter((n) => !/-(manual|pre-restore|uploaded)/.test(n)).length, 3);
assert.equal(left.filter((n) => n.includes("-manual")).length, 10);
assert.equal(left.filter((n) => n.includes("-pre-restore")).length, 3);
assert.equal(removed.length, 3 + 2 + 2);
assert.ok(left.includes("homelab-manager-2026-10-06-03-30-00.sqlite") && !left.includes("homelab-manager-2026-10-01-03-30-00.sqlite"), "the oldest went, not the newest");
// other files in the folder are left alone
writeFileSync(join(sb.backupDir(), "notes.txt"), "mine");
sb.pruneSnapshots(1);
assert.ok(existsSync(join(sb.backupDir(), "notes.txt")));
wipeBackups();
});
test("a scheduled run takes one and trims to what is kept, and records that it worked", async () => {
wipeBackups();
await updateSettings({ selfBackup: { keep: 2 } });
for (const d of ["2026-10-14T03:30:00Z", "2026-10-15T03:30:00Z", "2026-10-16T03:30:00Z"]) assert.ok(await runScheduledBackup(at(d)));
assert.deepEqual(names(), ["homelab-manager-2026-10-16-03-30-00.sqlite", "homelab-manager-2026-10-15-03-30-00.sqlite"]);
const last = await sb.lastAttempt();
assert.equal(last!.ok, true);
assert.equal(last!.kind, "scheduled");
assert.equal(cronForBackup("03:30"), "30 3 * * *");
assert.equal(cronForBackup("23:05"), "5 23 * * *");
await updateSettings({ selfBackup: { keep: 14 } });
});
test("a backup that can't be made is recorded as having failed, says why, and leaves no half file", async () => {
wipeBackups();
const realExecute = client.execute.bind(client);
(client as unknown as { execute: unknown }).execute = async (stmt: unknown) => {
if (typeof stmt === "string" && stmt.startsWith("VACUUM INTO")) throw new Error("SQLITE_FULL: database or disk is full");
return realExecute(stmt as never);
};
try {
await assert.rejects(sb.createSnapshot("scheduled", at("2026-10-17T03:30:00Z")), /SQLITE_FULL/);
const last = await sb.lastAttempt();
assert.equal(last!.ok, false);
assert.match(last!.error!, /disk is full/);
assert.deepEqual(sb.listSnapshots(), [], "nothing half-written is left in the backups folder");
assert.equal(await runScheduledBackup(at("2026-10-17T03:31:00Z")), null, "the scheduled run swallows it (and says so through the channels)");
} finally {
(client as unknown as { execute: unknown }).execute = realExecute;
}
assert.ok(await runScheduledBackup(at("2026-10-17T03:32:00Z")), "and the next one works");
assert.equal((await sb.lastAttempt())!.ok, true);
wipeBackups();
});
});
describe("restoring", () => {
test("a staged restore is checked first, kept beside the database, can be cancelled, and refuses what isn't usable", async () => {
wipeBackups();
const { info } = await sb.createSnapshot("manual", at("2026-10-17T15:00:00Z"));
assert.equal(sb.stagedRestore(), null);
await assert.rejects(sb.stageRestore("../../etc/passwd", "Ada"), /no backup with that name/);
await assert.rejects(sb.stageRestore("homelab-manager-2026-01-01-00-00-00.sqlite", "Ada"), /no backup with that name/);
writeFileSync(join(sb.backupDir(), "homelab-manager-2026-10-17-16-00-00-uploaded.sqlite"), "garbage");
await assert.rejects(sb.stageRestore("homelab-manager-2026-10-17-16-00-00-uploaded.sqlite", "Ada"), /isn't a database file/);
assert.equal(sb.stagedRestore(), null, "a refused one stages nothing");
const check = await sb.stageRestore(info.name, "Ada", at("2026-10-17T15:05:00Z"));
assert.equal(check.ok, true);
const staged = sb.stagedRestore()!;
assert.deepEqual(staged, { from: info.name, stagedAt: "2026-10-17T15:05:00.000Z", by: "Ada" });
const dataDir = join(dir, "data");
assert.ok(existsSync(join(dataDir, PENDING_FILE)));
assert.equal(readdirSync(dataDir).some((f) => f.endsWith(".copying")), false, "no half-copied file is left");
assert.equal(statSync(join(dataDir, PENDING_FILE)).size, info.sizeBytes);
assert.equal(sb.cancelStagedRestore(), true);
assert.equal(sb.stagedRestore(), null);
assert.equal(existsSync(join(dataDir, PENDING_FILE)), false);
assert.equal(sb.cancelStagedRestore(), false);
wipeBackups();
});
test("at start-up the staged file replaces the database, the old one is kept, and nothing happens twice", () => {
const folder = mkdtempSync(join(dir, "apply-"));
const dbFile = join(folder, "app.sqlite");
writeFileSync(dbFile, "OLD DATABASE");
writeFileSync(`${dbFile}-wal`, "stale journal");
writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE");
writeFileSync(join(folder, PENDING_NOTE), JSON.stringify({ from: "homelab-manager-2026-10-01-03-30-00.sqlite", by: "Ada" }));
const result = applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z"))!;
assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE");
assert.equal(existsSync(`${dbFile}-wal`), false, "a journal from the old file would be taken for the new one's");
assert.equal(result.keptAs, "homelab-manager-2026-10-17-16-00-00-pre-restore.sqlite");
assert.equal(readFileSync(join(folder, "backups", result.keptAs!), "utf8"), "OLD DATABASE", "what was in use is kept, not deleted");
assert.equal(result.from, "homelab-manager-2026-10-01-03-30-00.sqlite");
assert.equal(result.by, "Ada");
assert.equal(existsSync(join(folder, PENDING_FILE)), false);
assert.equal(existsSync(join(folder, PENDING_NOTE)), false);
assert.deepEqual(JSON.parse(readFileSync(join(folder, RESULT_NOTE), "utf8")), result);
assert.equal(applyPendingRestore(dbFile, at("2026-10-17T16:01:00Z")), null, "nothing staged any more");
assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE");
});
test("with no database there yet (a fresh machine), the staged one simply becomes it", () => {
const folder = mkdtempSync(join(dir, "fresh-"));
const dbFile = join(folder, "app.sqlite");
writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE");
const result = applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z"))!;
assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE");
assert.equal(result.keptAs, null);
assert.equal(result.from, "an uploaded backup", "no note, no name — said plainly");
});
test("a restore that can't be completed leaves the database exactly as it was", () => {
// the folder for keeping the old one can't be made (a file is in its place)
const folder = mkdtempSync(join(dir, "fail-"));
const dbFile = join(folder, "app.sqlite");
writeFileSync(dbFile, "OLD DATABASE");
writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE");
writeFileSync(join(folder, "backups"), "I am a file, not a folder");
assert.equal(applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z")), null);
assert.equal(readFileSync(dbFile, "utf8"), "OLD DATABASE", "the database is as it was");
assert.equal(readFileSync(join(folder, PENDING_FILE), "utf8"), "NEW DATABASE", "and the staged one is still there to try again");
assert.equal(existsSync(join(folder, RESULT_NOTE)), false);
// something staged that isn't a file is never put in the database's place
const odd = mkdtempSync(join(dir, "odd-"));
writeFileSync(join(odd, "app.sqlite"), "OLD DATABASE");
mkdirSync(join(odd, PENDING_FILE));
assert.equal(applyPendingRestore(join(odd, "app.sqlite"), at("2026-10-17T16:00:00Z")), null);
assert.equal(readFileSync(join(odd, "app.sqlite"), "utf8"), "OLD DATABASE");
});
});
describe("through the routes", () => {
let app: Server;
let base = "";
let who = "ada";
const call = (method: string, path: string, body?: unknown, raw?: Buffer) =>
fetch(`${base}${path}`, raw ? { method, headers: { "content-type": "application/octet-stream" }, body: new Uint8Array(raw) } : { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
before(async () => {
await db.insert(users).values({ oidcSub: "operator", name: "Olle", role: "operator" });
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/db-backups", dbBackupsRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/db-backups`;
wipeBackups();
});
after(() => app.close());
test("everything about backups is for administrators only", async () => {
who = "operator";
for (const [method, path] of [["GET", "/"], ["POST", "/"], ["PUT", "/settings"], ["POST", "/upload"], ["GET", "/x/download"], ["POST", "/x/restore"], ["POST", "/x/check"], ["DELETE", "/x"], ["POST", "/restore/cancel"]] as const) {
assert.equal((await call(method, path, method === "GET" || method === "DELETE" ? undefined : {})).status, 403, `${method} ${path}`);
}
who = "ada";
assert.equal((await call("GET", "/")).status, 200);
});
test("settings are checked and kept, and change what is kept at once", async () => {
assert.equal((await call("PUT", "/settings", { enabled: true, time: "25:00", keep: 7 })).status, 400);
assert.equal((await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 0 })).status, 400);
assert.equal((await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 500 })).status, 400);
const res = await call("PUT", "/settings", { enabled: true, time: "04:15", keep: 7 });
assert.equal(res.status, 200);
const body = await res.json();
assert.deepEqual(body.settings, { enabled: true, time: "04:15", keep: 7 });
assert.ok(body.nextRun, "it says when the next one is");
assert.equal((await getSettings()).selfBackup.time, "04:15");
const off = await (await call("PUT", "/settings", { enabled: false, time: "04:15", keep: 7 })).json();
assert.equal(off.nextRun, null, "off means nothing is scheduled");
await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 14 });
const entry = (await db.select().from(auditLog)).filter((e) => e.category === "db_backup" && e.action === "settings").pop()!;
assert.deepEqual(JSON.parse(entry.detail!).to, { enabled: true, time: "03:30", keep: 14 });
});
test("take one, see it listed, download it byte for byte, check it, and delete it", async () => {
const made = await call("POST", "/");
assert.equal(made.status, 201);
const { snapshot, check } = await made.json();
assert.equal(check.ok, true);
assert.match(snapshot.name, /-manual\.sqlite$/);
const overview = await (await call("GET", "/")).json();
assert.equal(overview.snapshots.length, 1);
assert.equal(overview.snapshots[0].name, snapshot.name);
assert.equal(overview.lastAttempt.ok, true);
assert.equal(overview.staged, null);
const download = await call("GET", `/${snapshot.name}/download`);
assert.equal(download.status, 200);
assert.equal(download.headers.get("cache-control"), "no-store");
assert.match(download.headers.get("content-disposition") ?? "", new RegExp(snapshot.name.replace(/\./g, "\\.")));
const bytes = Buffer.from(await download.arrayBuffer());
assert.deepEqual(bytes, readFileSync(join(sb.backupDir(), snapshot.name)));
assert.equal((await (await call("POST", `/${snapshot.name}/check`)).json()).check.integrity, "ok");
assert.equal((await call("GET", "/homelab-manager-nonsense.sqlite/download")).status, 404);
assert.equal((await call("GET", `/..%2Fapp.sqlite/download`)).status, 404);
assert.equal((await call("DELETE", `/${snapshot.name}`)).status, 204);
assert.equal((await call("DELETE", `/${snapshot.name}`)).status, 404);
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "db_backup").map((e) => e.action);
for (const wanted of ["create", "download", "delete"]) assert.ok(actions.includes(wanted), wanted);
});
test("a backup from elsewhere can be uploaded and then restored like any other; a bad file is refused and leaves nothing", async () => {
const { info } = await sb.createSnapshot("manual", at("2026-10-17T18:00:00Z"));
const bytes = readFileSync(join(sb.backupDir(), info.name));
const before = sb.listSnapshots().length;
const up = await call("POST", "/upload", undefined, bytes);
assert.equal(up.status, 201);
const uploaded = (await up.json()).snapshot;
assert.match(uploaded.name, /-uploaded\.sqlite$/);
assert.equal(sb.listSnapshots().length, before + 1);
const bad = await call("POST", "/upload", undefined, Buffer.from("not a database"));
assert.equal(bad.status, 400);
assert.match((await bad.json()).message, /isn't a database file/);
assert.equal((await call("POST", "/upload", undefined, Buffer.alloc(0))).status, 400);
assert.equal(sb.listSnapshots().length, before + 1, "nothing was kept from the bad ones");
const restore = await call("POST", `/${uploaded.name}/restore`);
assert.equal(restore.status, 200);
assert.equal((await restore.json()).staged.from, uploaded.name);
assert.equal((await (await call("GET", "/")).json()).staged.from, uploaded.name);
assert.equal((await call("POST", "/homelab-manager-2026-01-01-00-00-00.sqlite/restore")).status, 400);
assert.equal((await (await call("POST", "/restore/cancel")).json()).cancelled, true);
assert.equal((await (await call("GET", "/")).json()).staged, null);
assert.ok((await db.select().from(auditLog)).some((e) => e.action === "restore_staged"));
assert.ok((await db.select().from(auditLog)).some((e) => e.action === "restore_cancelled"));
});
test("what the last restore did is shown until it is dismissed", async () => {
writeFileSync(join(dir, "data", RESULT_NOTE), JSON.stringify({ restoredAt: "2026-10-17T16:00:00.000Z", from: "x.sqlite", keptAs: "y.sqlite", by: "Ada" }));
assert.equal((await (await call("GET", "/")).json()).lastRestore.keptAs, "y.sqlite");
assert.equal((await call("POST", "/restore-result/dismiss")).status, 200);
assert.equal((await (await call("GET", "/")).json()).lastRestore, null);
});
});
describe("how current the backups are, in the alerts and /metrics", () => {
const selfAlert = async () => {
invalidateAlertsCache();
return (await getAlerts(true)).alerts.find((a) => a.id === "backup:self-backup");
};
const oneAged = async (hoursAgo: number) => {
wipeBackups();
const { info } = await sb.createSnapshot("scheduled", new Date());
const when = new Date(Date.now() - hoursAgo * 3600_000);
utimesSync(join(sb.backupDir(), info.name), when, when);
};
test("no alert while they are recent, or when none was ever tried; a warning when stale, critical after a week", async () => {
wipeBackups();
assert.equal(await selfAlert(), undefined, "nothing yet, and nothing failed: a new install isn't scolded");
await oneAged(5);
assert.equal(await selfAlert(), undefined);
await oneAged(60);
const stale = await selfAlert();
assert.equal(stale!.severity, "warning");
assert.match(stale!.message, /60 h old/);
assert.equal(stale!.source, "Homelab Manager");
assert.equal(stale!.link, "/settings/backup");
await oneAged(24 * 8);
assert.equal((await selfAlert())!.severity, "critical");
assert.match((await selfAlert())!.message, /8 days old/);
});
test("when the last attempt failed, the alert says why; and nothing is said at all while backups are switched off", async () => {
await oneAged(60);
await setInternalFlag("selfBackupLastAttempt", JSON.stringify({ at: new Date().toISOString(), ok: false, kind: "scheduled", error: "ENOSPC: no space left on device", name: null }));
assert.match((await selfAlert())!.message, /The last attempt failed: ENOSPC: no space left on device/);
wipeBackups();
const first = await selfAlert();
assert.match(first!.message, /no backup of its own yet\. The last attempt failed|no backup of its own yet/, "a failure with nothing to show for it is said");
await updateSettings({ selfBackup: { enabled: false } });
assert.equal(await selfAlert(), undefined);
await updateSettings({ selfBackup: { enabled: true } });
});
test("/metrics says how many are kept, when the newest was made and whether the last try worked", async () => {
await oneAged(2);
await sb.createSnapshot("manual", new Date());
const text = await buildMetrics();
assert.match(text, /homelab_selfbackup_snapshots 2/);
const ts = /homelab_selfbackup_last_success_timestamp_seconds (\d+)/.exec(text);
assert.ok(ts && Math.abs(Number(ts[1]) - Date.now() / 1000) < 120, "the manual one just made is the newest");
assert.match(text, /homelab_selfbackup_last_attempt_ok 1/);
wipeBackups();
});
});
+3 -3
View File
@@ -282,9 +282,9 @@ describe("following a run to the end", () => {
const titles = events.filter((e) => /Update run/.test(e.title)).map((e) => e.title);
// (the tests before this one wrote their own lines too, so this looks for the kinds rather than the number)
for (const wanted of ["Update run started", "Update run didn't finish", "Update run finished", "Update run stopped being followed"]) assert.ok(titles.includes(wanted), `${wanted} in ${JSON.stringify([...new Set(titles)])}`);
const failedLine = events.find((e) => e.title === "Update run didn't finish")!;
assert.equal(failedLine.tone, "bad");
assert.match(failedLine.detail!, /The playbook failed in Semaphore/);
const failedLines = events.filter((e) => e.title === "Update run didn't finish");
assert.ok(failedLines.every((e) => e.tone === "bad"), "a run that didn't finish reads as bad");
assert.ok(failedLines.some((e) => /The playbook failed in Semaphore/.test(e.detail ?? "")), "and says why");
assert.ok((await db.select().from(auditLog)).some((a) => a.category === "update_run" && a.action === "failed"));
// nothing is held in memory: a fresh pass over the table is all it takes