Acknowledge and hand over alerts, incident timelines, and built-in database backups
- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics - Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents, incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs, related alerts and notes; Incidents page and a digest section - Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download, check, upload, delete, and restore staged and applied at start-up before the database opens (pre-restore copy kept); alerts and /metrics for stale or failing backups - Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7a5b623444
commit
2744a84719
41 files changed
+6573
-26
No files matched your search
@@ -0,0 +1,46 @@
|
||||
CREATE TABLE `alert_acks` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`alert_key` text NOT NULL,
|
||||
`note` text DEFAULT '' NOT NULL,
|
||||
`assignee_user_id` integer,
|
||||
`assignee_name` text,
|
||||
`acked_by` text NOT NULL,
|
||||
`acked_at` text NOT NULL,
|
||||
`until` text,
|
||||
`last_seen_at` text NOT NULL,
|
||||
FOREIGN KEY (`assignee_user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE set null
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX `alert_acks_key_idx` ON `alert_acks` (`alert_key`);--> statement-breakpoint
|
||||
CREATE TABLE `alert_incidents` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`alert_key` text NOT NULL,
|
||||
`severity` text NOT NULL,
|
||||
`peak_severity` text NOT NULL,
|
||||
`category` text NOT NULL,
|
||||
`source` text NOT NULL,
|
||||
`message` text NOT NULL,
|
||||
`link` text,
|
||||
`target_type` text,
|
||||
`target_id` integer,
|
||||
`target_name` text,
|
||||
`caused_by` text,
|
||||
`opened_at` text NOT NULL,
|
||||
`resolved_at` text,
|
||||
`last_seen_at` text NOT NULL,
|
||||
`missing_since` text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `alert_incidents_key_idx` ON `alert_incidents` (`alert_key`);--> statement-breakpoint
|
||||
CREATE INDEX `alert_incidents_opened_idx` ON `alert_incidents` (`opened_at`);--> statement-breakpoint
|
||||
CREATE INDEX `alert_incidents_target_idx` ON `alert_incidents` (`target_type`,`target_id`);--> statement-breakpoint
|
||||
CREATE TABLE `incident_notes` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`incident_id` integer NOT NULL,
|
||||
`author` text NOT NULL,
|
||||
`body` text NOT NULL,
|
||||
`created_at` text NOT NULL,
|
||||
FOREIGN KEY (`incident_id`) REFERENCES `alert_incidents`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `incident_notes_incident_idx` ON `incident_notes` (`incident_id`);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -225,6 +225,13 @@
|
||||
"when": 1792278718505,
|
||||
"tag": "0031_sad_lizard",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 32,
|
||||
"version": "6",
|
||||
"when": 1792281947686,
|
||||
"tag": "0032_daffy_captain_stacy",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -12,6 +12,8 @@ import { uptimeRouter } from "./routes/uptime.js";
|
||||
import { shareLinksRouter } from "./routes/shareLinks.js";
|
||||
import { runbooksRouter } from "./routes/runbooks.js";
|
||||
import { updateRunsRouter } from "./routes/updateRuns.js";
|
||||
import { dbBackupsRouter } from "./routes/dbBackups.js";
|
||||
import { incidentsRouter } from "./routes/incidents.js";
|
||||
import { inventoryRouter } from "./routes/inventory.js";
|
||||
import { backupsRouter } from "./routes/backups.js";
|
||||
import { metricsRouter, metricsTokensRouter } from "./routes/metrics.js";
|
||||
@@ -100,6 +102,8 @@ export function createApp() {
|
||||
app.use("/api/share-links", shareLinksRouter);
|
||||
app.use("/api/runbooks", runbooksRouter);
|
||||
app.use("/api/update-runs", updateRunsRouter);
|
||||
app.use("/api/db-backups", dbBackupsRouter);
|
||||
app.use("/api/incidents", incidentsRouter);
|
||||
app.use("/api/inventory", inventoryRouter);
|
||||
app.use("/api/backups", backupsRouter);
|
||||
app.use("/api/metrics-tokens", metricsTokensRouter);
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* Finishes a restore that was staged from Settings → Backup. It has to happen before the database is opened, which is why it is a module
|
||||
* of its own that index.ts imports first: by the time anything else loads, the file is already the one that was chosen.
|
||||
*
|
||||
* Nothing is deleted. The database that was in use is moved into the backups folder (as "…-pre-restore.sqlite") before the chosen one is put
|
||||
* in its place, so a restore that wasn't what was wanted can itself be undone by restoring that file. If anything goes wrong half way, the
|
||||
* old file is put back and the staged one is left where it is.
|
||||
*/
|
||||
import "../loadEnv.js";
|
||||
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { resolveDataPath } from "../paths.js";
|
||||
|
||||
import { PENDING_FILE, PENDING_NOTE, RESULT_NOTE, type RestoreResult } from "./restoreFiles.js";
|
||||
|
||||
const stamp = (d: Date) => d.toISOString().slice(0, 19).replace(/[:T]/g, "-");
|
||||
|
||||
/** Puts a staged database in place, if one is waiting. Returns what it did, or null if nothing was staged. */
|
||||
export function applyPendingRestore(dbFile: string, now: Date = new Date()): RestoreResult | null {
|
||||
const dataDir = dirname(dbFile);
|
||||
const pending = join(dataDir, PENDING_FILE);
|
||||
if (!existsSync(pending)) return null;
|
||||
// Only a file is ever put in place of the database.
|
||||
if (!statSync(pending).isFile()) {
|
||||
console.error(`[restore] ${PENDING_FILE} isn't a file, so it was left alone and the database is as it was.`);
|
||||
return null;
|
||||
}
|
||||
|
||||
let note: { from?: string; by?: string | null } = {};
|
||||
try {
|
||||
note = JSON.parse(readFileSync(join(dataDir, PENDING_NOTE), "utf8"));
|
||||
} catch {
|
||||
// the note only says where it came from; the file is what matters
|
||||
}
|
||||
|
||||
const backups = join(dataDir, "backups");
|
||||
let keptAs: string | null = null;
|
||||
let moved = false;
|
||||
try {
|
||||
mkdirSync(backups, { recursive: true });
|
||||
if (existsSync(dbFile)) {
|
||||
let name = `homelab-manager-${stamp(now)}-pre-restore.sqlite`;
|
||||
for (let n = 2; existsSync(join(backups, name)); n++) name = `homelab-manager-${stamp(now)}-pre-restore-${n}.sqlite`;
|
||||
renameSync(dbFile, join(backups, name));
|
||||
keptAs = name;
|
||||
moved = true;
|
||||
}
|
||||
// A journal left by the old file would be taken for the new one's.
|
||||
for (const suffix of ["-wal", "-shm", "-journal"]) rmSync(dbFile + suffix, { force: true });
|
||||
renameSync(pending, dbFile);
|
||||
} catch (err) {
|
||||
if (moved && keptAs && !existsSync(dbFile)) {
|
||||
try {
|
||||
renameSync(join(backups, keptAs), dbFile);
|
||||
} catch {
|
||||
// nothing more can be done; the old file is still in the backups folder under its new name
|
||||
}
|
||||
}
|
||||
console.error("[restore] couldn't put the staged backup in place; the database is as it was:", err instanceof Error ? err.message : err);
|
||||
return null;
|
||||
}
|
||||
|
||||
const result: RestoreResult = { restoredAt: now.toISOString(), from: note.from ?? "an uploaded backup", keptAs, by: note.by ?? null };
|
||||
writeFileSync(join(dataDir, RESULT_NOTE), JSON.stringify(result));
|
||||
rmSync(join(dataDir, PENDING_NOTE), { force: true });
|
||||
console.log(`[restore] restored the database from ${result.from}${keptAs ? `; the one that was in use is kept as backups/${keptAs}` : ""}.`);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Run it now, on the database this process is about to open.
|
||||
applyPendingRestore(resolveDataPath(process.env.DATABASE_PATH ?? "../data/homelab-manager.sqlite"));
|
||||
@@ -12,4 +12,4 @@ const client = createClient({ url: `file:${dbPath}` });
|
||||
await client.execute("PRAGMA foreign_keys = ON;");
|
||||
|
||||
export const db = drizzle(client, { schema });
|
||||
export { client };
|
||||
export { client, dbPath };
|
||||
@@ -0,0 +1,13 @@
|
||||
// The names a staged restore uses in the data folder. Kept apart from applyRestore.ts, which does something when it is imported.
|
||||
export const PENDING_FILE = "restore-pending.sqlite";
|
||||
export const PENDING_NOTE = "restore-pending.json";
|
||||
export const RESULT_NOTE = "restore-result.json";
|
||||
|
||||
export interface RestoreResult {
|
||||
restoredAt: string;
|
||||
/** The backup that is now the database. */
|
||||
from: string;
|
||||
/** Where the database that was in use went. */
|
||||
keptAs: string | null;
|
||||
by: string | null;
|
||||
}
|
||||
@@ -572,6 +572,63 @@ export const runbooks = sqliteTable(
|
||||
(t) => [index("runbooks_target_idx").on(t.targetType, t.targetId)],
|
||||
);
|
||||
|
||||
// ─── Acknowledgements and incidents ─────────────────────────────────────────
|
||||
|
||||
// "I'm on it" for one alert — see services/alertAcks.ts. Keyed by the alert's stable id (not a foreign key to anything); gone once the alert has been.
|
||||
export const alertAcks = sqliteTable(
|
||||
"alert_acks",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
alertKey: text("alert_key").notNull(),
|
||||
note: text("note").notNull().default(""),
|
||||
assigneeUserId: integer("assignee_user_id").references(() => users.id, { onDelete: "set null" }),
|
||||
assigneeName: text("assignee_name"), // as it was when handed over, so it still reads if the user is removed
|
||||
ackedBy: text("acked_by").notNull(),
|
||||
ackedAt: text("acked_at").notNull(), // ISO
|
||||
until: text("until"), // ISO; null = until the alert is gone
|
||||
lastSeenAt: text("last_seen_at").notNull(), // ISO; the last time the alert was there
|
||||
},
|
||||
(t) => [uniqueIndex("alert_acks_key_idx").on(t.alertKey)],
|
||||
);
|
||||
|
||||
// One time an alert was raised and later went away — see services/incidents.ts. When the same alert comes back it is a new row.
|
||||
export const alertIncidents = sqliteTable(
|
||||
"alert_incidents",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
alertKey: text("alert_key").notNull(),
|
||||
severity: text("severity").notNull(), // as it is now (or was when it ended)
|
||||
peakSeverity: text("peak_severity").notNull(),
|
||||
category: text("category").notNull(),
|
||||
source: text("source").notNull(),
|
||||
message: text("message").notNull(), // the latest wording
|
||||
link: text("link"),
|
||||
targetType: text("target_type"), // server | integration | dns_provider, when the alert is about one of those
|
||||
targetId: integer("target_id"),
|
||||
targetName: text("target_name"),
|
||||
causedBy: text("caused_by"), // the host whose outage this was folded under
|
||||
openedAt: text("opened_at").notNull(), // ISO
|
||||
resolvedAt: text("resolved_at"), // ISO; null while it is open
|
||||
lastSeenAt: text("last_seen_at").notNull(), // ISO
|
||||
missingSince: text("missing_since"), // ISO; set while it has been absent from the last look(s), before it is called over
|
||||
},
|
||||
(t) => [index("alert_incidents_key_idx").on(t.alertKey), index("alert_incidents_opened_idx").on(t.openedAt), index("alert_incidents_target_idx").on(t.targetType, t.targetId)],
|
||||
);
|
||||
|
||||
export const incidentNotes = sqliteTable(
|
||||
"incident_notes",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
incidentId: integer("incident_id")
|
||||
.notNull()
|
||||
.references(() => alertIncidents.id, { onDelete: "cascade" }),
|
||||
author: text("author").notNull(),
|
||||
body: text("body").notNull(),
|
||||
createdAt: text("created_at").notNull(), // ISO
|
||||
},
|
||||
(t) => [index("incident_notes_incident_idx").on(t.incidentId)],
|
||||
);
|
||||
|
||||
// ─── Update runs: apply a server's updates and see it through ──────────────
|
||||
|
||||
// A playbook run in Semaphore for one server, wrapped in a maintenance window and followed until the server reports again — see
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import "./loadEnv.js";
|
||||
// Before anything opens the database: puts a restore that was staged from Settings → Backup in place.
|
||||
import "./db/applyRestore.js";
|
||||
import { env, warnIfAuthNotConfigured, checkSessionSecret } from "./env.js";
|
||||
import { createApp } from "./app.js";
|
||||
import { runMigrations } from "./db/migrate.js";
|
||||
@@ -17,6 +19,9 @@ import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
|
||||
import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceScheduler.js";
|
||||
import { initMaintenanceScheduleScheduler } from "./services/maintenanceScheduleScheduler.js";
|
||||
import { initUpdateRuns } from "./services/updateRuns.js";
|
||||
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
|
||||
import { initIncidentRecorder } from "./services/incidents.js";
|
||||
import { getAlerts } from "./services/alerts.js";
|
||||
|
||||
{
|
||||
// Sessions are signed with this secret. Without one, the app would sign them with a value that's in the source code.
|
||||
@@ -50,6 +55,8 @@ await initAlertTicketScheduler();
|
||||
await initKumaMaintenanceScheduler();
|
||||
await initMaintenanceScheduleScheduler();
|
||||
initUpdateRuns();
|
||||
await initSelfBackupScheduler();
|
||||
initIncidentRecorder(() => getAlerts(false));
|
||||
|
||||
createApp().listen(env.port, () => {
|
||||
console.log(`homelab-manager listening on port ${env.port}`);
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { z } from "zod";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { getAlerts } from "../services/alerts.js";
|
||||
import { AckError, assignableUsers, attentionCounts, clearAck, loadAcks, MAX_HOURS, MAX_NOTE, setAck } from "../services/alertAcks.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { openIncidentIds } from "../services/incidents.js";
|
||||
import { ticketsForAlerts, type AlertTicketInfo } from "../services/alertTickets.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
@@ -19,5 +23,68 @@ alertsRouter.get("/", asyncHandler(async (req, res) => {
|
||||
} catch (err) {
|
||||
console.error("[alerts] couldn't look up tickets:", err);
|
||||
}
|
||||
res.json({ ...report, alerts: report.alerts.map((a) => ({ ...a, ticket: tickets.get(a.id) ?? null })) });
|
||||
// Who has said they are on it, and the incident record each alert is part of. Neither may stop the page from loading.
|
||||
let acks = new Map<string, Awaited<ReturnType<typeof loadAcks>> extends Map<string, infer V> ? V : never>();
|
||||
let incidents = new Map<string, number>();
|
||||
try {
|
||||
acks = await loadAcks();
|
||||
incidents = await openIncidentIds(report.alerts.map((a) => a.id));
|
||||
} catch (err) {
|
||||
console.error("[alerts] couldn't look up acknowledgements:", err);
|
||||
}
|
||||
const attention = attentionCounts(report.alerts, acks);
|
||||
res.json({
|
||||
...report,
|
||||
// `counts` still counts everything, as /metrics and the wall do; `attention` leaves out what somebody has said they are dealing with.
|
||||
counts: { ...report.counts, acknowledged: attention.acknowledged, attention: { critical: attention.critical, warning: attention.warning } },
|
||||
alerts: report.alerts.map((a) => ({ ...a, ticket: tickets.get(a.id) ?? null, ack: acks.get(a.id) ?? null, incidentId: incidents.get(a.id) ?? null })),
|
||||
});
|
||||
}));
|
||||
|
||||
// Who an alert can be handed to.
|
||||
alertsRouter.get("/assignees", requireRole("operator"), asyncHandler(async (_req, res) => {
|
||||
res.json({ users: await assignableUsers() });
|
||||
}));
|
||||
|
||||
const ackSchema = z.object({
|
||||
alertId: z.string().min(1).max(600),
|
||||
note: z.string().max(MAX_NOTE).default(""),
|
||||
assigneeId: z.number().int().positive().nullable().default(null),
|
||||
/** How long it lasts; null (or left out) is until the alert is gone. */
|
||||
hours: z.number().int().min(1).max(MAX_HOURS).nullable().default(null),
|
||||
});
|
||||
|
||||
const auditTargetOf = (alert: { target: { type: string; id: number } | null }) => (alert.target ? { targetType: alert.target.type, targetId: alert.target.id } : {});
|
||||
|
||||
// "I'm on it": acknowledge an alert, say why, hand it to someone, and say how long it should be left alone.
|
||||
alertsRouter.put("/ack", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = ackSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "Say which alert, and optionally why, who has it and for how long.", details: parsed.error.flatten() });
|
||||
const alert = (await getAlerts(false)).alerts.find((a) => a.id === parsed.data.alertId);
|
||||
if (!alert) return res.status(404).json({ error: "not_found", message: "That alert isn't there any more." });
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
const { ack, before } = await setAck({ ...parsed.data, actor: actor.name ?? actor.email ?? actor.oidcSub });
|
||||
await recordAudit({
|
||||
actor,
|
||||
category: "alert",
|
||||
action: "ack",
|
||||
...auditTargetOf(alert),
|
||||
detail: { alertId: alert.id, message: alert.message, note: ack.note, assignee: ack.assignee?.name ?? null, hours: parsed.data.hours, changed: before !== null },
|
||||
});
|
||||
res.json({ ack });
|
||||
} catch (err) {
|
||||
if (err instanceof AckError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
alertsRouter.delete("/ack", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const alertId = typeof req.query.alertId === "string" ? req.query.alertId : "";
|
||||
if (!alertId) return res.status(400).json({ error: "invalid_id" });
|
||||
const removed = await clearAck(alertId);
|
||||
if (!removed) return res.status(404).json({ error: "not_found" });
|
||||
const alert = (await getAlerts(false)).alerts.find((a) => a.id === alertId);
|
||||
await recordAudit({ actor: req.currentUser!, category: "alert", action: "unack", ...(alert ? auditTargetOf(alert) : {}), detail: { alertId, message: alert?.message ?? null } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -0,0 +1,135 @@
|
||||
import express, { Router } from "express";
|
||||
import { z } from "zod";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
import {
|
||||
backupDir,
|
||||
cancelStagedRestore,
|
||||
checkSnapshotFile,
|
||||
createSnapshot,
|
||||
deleteSnapshot,
|
||||
dismissLastRestore,
|
||||
importUploaded,
|
||||
lastAttempt,
|
||||
lastRestore,
|
||||
listSnapshots,
|
||||
pruneSnapshots,
|
||||
snapshotPath,
|
||||
stageRestore,
|
||||
stagedRestore,
|
||||
} from "../services/selfBackup.js";
|
||||
import { nextBackupRun, scheduleSelfBackup } from "../services/selfBackupScheduler.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const dbBackupsRouter = Router();
|
||||
|
||||
// A backup holds everything the app knows — the audit log, every server's address, the hashed agent tokens and all the stored credentials
|
||||
// (encrypted) — so all of it, looking and doing, is for administrators.
|
||||
dbBackupsRouter.use(requireAuth, requireRole("admin"));
|
||||
|
||||
/** A database file is far bigger than anything else this app is sent, and is only ever read from an administrator. */
|
||||
const UPLOAD_LIMIT = "1gb";
|
||||
|
||||
const settingsSchema = z.object({
|
||||
enabled: z.boolean(),
|
||||
time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/, "Use a time like 03:30"),
|
||||
keep: z.number().int().min(1).max(90),
|
||||
});
|
||||
|
||||
const actorName = (req: express.Request) => req.currentUser!.name ?? req.currentUser!.email ?? req.currentUser!.oidcSub;
|
||||
|
||||
async function overview() {
|
||||
const { selfBackup } = await getSettings();
|
||||
return {
|
||||
settings: selfBackup,
|
||||
snapshots: listSnapshots(),
|
||||
staged: stagedRestore(),
|
||||
lastRestore: lastRestore(),
|
||||
lastAttempt: await lastAttempt(),
|
||||
nextRun: nextBackupRun()?.toISOString() ?? null,
|
||||
folder: backupDir(),
|
||||
};
|
||||
}
|
||||
|
||||
dbBackupsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
res.json(await overview());
|
||||
}));
|
||||
|
||||
dbBackupsRouter.put("/settings", asyncHandler(async (req, res) => {
|
||||
const parsed = settingsSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "A time and how many to keep.", details: parsed.error.flatten() });
|
||||
const before = (await getSettings()).selfBackup;
|
||||
await updateSettings({ selfBackup: parsed.data });
|
||||
await scheduleSelfBackup();
|
||||
pruneSnapshots(parsed.data.keep);
|
||||
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "settings", targetType: "settings", detail: { from: before, to: parsed.data } });
|
||||
res.json(await overview());
|
||||
}));
|
||||
|
||||
// Take one now. It is checked as it is made, and pruned like the scheduled ones.
|
||||
dbBackupsRouter.post("/", asyncHandler(async (req, res) => {
|
||||
try {
|
||||
const { info, check } = await createSnapshot("manual");
|
||||
pruneSnapshots((await getSettings()).selfBackup.keep);
|
||||
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "create", targetType: "db_backup", targetId: info.name, detail: { sizeBytes: info.sizeBytes } });
|
||||
res.status(201).json({ snapshot: info, check });
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: "backup_failed", message: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
}));
|
||||
|
||||
// A backup somebody has (from another machine, from the script in BACKUP.md): kept if it checks out, and can then be restored like any other.
|
||||
dbBackupsRouter.post("/upload", express.raw({ type: () => true, limit: UPLOAD_LIMIT }), asyncHandler(async (req, res) => {
|
||||
if (!Buffer.isBuffer(req.body) || req.body.length === 0) return res.status(400).json({ error: "no_file", message: "Choose a backup file to upload." });
|
||||
try {
|
||||
const { info, check } = await importUploaded(req.body);
|
||||
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "upload", targetType: "db_backup", targetId: info.name, detail: { sizeBytes: info.sizeBytes } });
|
||||
res.status(201).json({ snapshot: info, check });
|
||||
} catch (err) {
|
||||
res.status(400).json({ error: "unusable", message: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
}));
|
||||
|
||||
dbBackupsRouter.post("/restore/cancel", asyncHandler(async (req, res) => {
|
||||
const had = cancelStagedRestore();
|
||||
if (had) await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "restore_cancelled" });
|
||||
res.json({ cancelled: had });
|
||||
}));
|
||||
|
||||
dbBackupsRouter.post("/restore-result/dismiss", asyncHandler(async (_req, res) => {
|
||||
dismissLastRestore();
|
||||
res.json({ ok: true });
|
||||
}));
|
||||
|
||||
dbBackupsRouter.post("/:name/check", asyncHandler(async (req, res) => {
|
||||
const file = snapshotPath(req.params.name);
|
||||
if (!file) return res.status(404).json({ error: "not_found" });
|
||||
res.json({ check: await checkSnapshotFile(file) });
|
||||
}));
|
||||
|
||||
dbBackupsRouter.get("/:name/download", asyncHandler(async (req, res) => {
|
||||
const file = snapshotPath(req.params.name);
|
||||
if (!file) return res.status(404).json({ error: "not_found" });
|
||||
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "download", targetType: "db_backup", targetId: req.params.name });
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
res.download(file, req.params.name);
|
||||
}));
|
||||
|
||||
// Checks the backup and stages it to replace the database the next time the app starts. Nothing changes until then, and the database in
|
||||
// use is kept (as a "pre-restore" backup) when it does.
|
||||
dbBackupsRouter.post("/:name/restore", asyncHandler(async (req, res) => {
|
||||
try {
|
||||
const check = await stageRestore(req.params.name, actorName(req));
|
||||
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "restore_staged", targetType: "db_backup", targetId: req.params.name });
|
||||
res.json({ staged: stagedRestore(), check });
|
||||
} catch (err) {
|
||||
res.status(400).json({ error: "cannot_restore", message: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
}));
|
||||
|
||||
dbBackupsRouter.delete("/:name", asyncHandler(async (req, res) => {
|
||||
if (!deleteSnapshot(req.params.name)) return res.status(404).json({ error: "not_found" });
|
||||
await recordAudit({ actor: req.currentUser!, category: "db_backup", action: "delete", targetType: "db_backup", targetId: req.params.name });
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -0,0 +1,51 @@
|
||||
import { Router } from "express";
|
||||
import { z } from "zod";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { addNote, getIncident, listIncidents, MAX_NOTE, removeNote } from "../services/incidents.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const incidentsRouter = Router();
|
||||
|
||||
// Everyone signed in can read what happened — it is made of what the Alerts page, the server pages and the Maintenance page already show.
|
||||
// Writing a note is for operators.
|
||||
incidentsRouter.use(requireAuth);
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
incidentsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
res.json({ incidents: await listIncidents() });
|
||||
}));
|
||||
|
||||
incidentsRouter.get("/:id", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const detail = await getIncident(id);
|
||||
if (!detail) return res.status(404).json({ error: "not_found" });
|
||||
res.json(detail);
|
||||
}));
|
||||
|
||||
const noteSchema = z.object({ body: z.string().trim().min(1, "Write something first.").max(MAX_NOTE) });
|
||||
|
||||
incidentsRouter.post("/:id/notes", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = noteSchema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.success ? undefined : parsed.error.issues[0]?.message });
|
||||
const actor = req.currentUser!;
|
||||
const note = await addNote(id, actor.name ?? actor.email ?? actor.oidcSub, parsed.data.body);
|
||||
if (!note) return res.status(404).json({ error: "not_found" });
|
||||
await recordAudit({ actor, category: "incident", action: "note", targetType: "incident", targetId: id });
|
||||
res.status(201).json({ id: note.id });
|
||||
}));
|
||||
|
||||
incidentsRouter.delete("/:id/notes/:noteId", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const noteId = idOf(req.params.noteId);
|
||||
if (!Number.isInteger(id) || !Number.isInteger(noteId)) return res.status(400).json({ error: "invalid_id" });
|
||||
const actor = req.currentUser!;
|
||||
const outcome = await removeNote(id, noteId, { name: actor.name ?? actor.email ?? actor.oidcSub, isAdmin: actor.role === "admin" });
|
||||
if (outcome === "not_found") return res.status(404).json({ error: "not_found" });
|
||||
if (outcome === "forbidden") return res.status(403).json({ error: "forbidden", message: "A note can be removed by the person who wrote it, or an administrator." });
|
||||
await recordAudit({ actor, category: "incident", action: "note_removed", targetType: "incident", targetId: id });
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* "I'm on it": an acknowledgement says that somebody has seen an alert, may say who is dealing with it and why, and may run out after a
|
||||
* while. It changes nothing about the problem — the alert is still raised, still counted by /metrics and still on the wall display — but
|
||||
* the people looking at the app can tell what nobody has looked at yet from what somebody has, and the sidebar and dashboard count only the
|
||||
* former.
|
||||
*
|
||||
* An acknowledgement belongs to the alert's id (the same stable key the Alerts page keys rows on). It goes away on its own once the alert
|
||||
* has been gone for a while, so the same problem coming back later is a new alert that nobody has looked at, not one that is "still being
|
||||
* dealt with" from last month.
|
||||
*/
|
||||
import { and, eq, inArray, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { alertAcks, users } from "../db/schema.js";
|
||||
import type { Alert } from "./alertTypes.js";
|
||||
|
||||
export interface AckInfo {
|
||||
note: string;
|
||||
assignee: { id: number | null; name: string } | null;
|
||||
ackedBy: string;
|
||||
ackedAt: string;
|
||||
/** ISO — when it runs out; null means it lasts until the alert is gone. */
|
||||
until: string | null;
|
||||
/** Still in force: has no end, or the end hasn't come. */
|
||||
active: boolean;
|
||||
}
|
||||
|
||||
/** An alert that has been absent this long loses its acknowledgement (a blip of a few minutes doesn't). */
|
||||
export const GONE_AFTER_MS = 30 * 60_000;
|
||||
export const MAX_NOTE = 500;
|
||||
export const MAX_HOURS = 24 * 30;
|
||||
|
||||
export class AckError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly code: string,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
const shape = (r: typeof alertAcks.$inferSelect, now: Date): AckInfo => ({
|
||||
note: r.note,
|
||||
assignee: r.assigneeName ? { id: r.assigneeUserId, name: r.assigneeName } : null,
|
||||
ackedBy: r.ackedBy,
|
||||
ackedAt: r.ackedAt,
|
||||
until: r.until,
|
||||
active: r.until === null || Date.parse(r.until) > now.getTime(),
|
||||
});
|
||||
|
||||
export async function loadAcks(now: Date = new Date()): Promise<Map<string, AckInfo>> {
|
||||
const rows = await db.select().from(alertAcks);
|
||||
return new Map(rows.map((r) => [r.alertKey, shape(r, now)]));
|
||||
}
|
||||
|
||||
export async function loadAck(alertKey: string, now: Date = new Date()): Promise<AckInfo | null> {
|
||||
const [row] = await db.select().from(alertAcks).where(eq(alertAcks.alertKey, alertKey)).limit(1);
|
||||
return row ? shape(row, now) : null;
|
||||
}
|
||||
|
||||
export interface SetAckInput {
|
||||
alertId: string;
|
||||
note: string;
|
||||
/** A user to hand it to; null for nobody in particular. */
|
||||
assigneeId: number | null;
|
||||
/** How long the acknowledgement lasts; null means until the alert is gone. */
|
||||
hours: number | null;
|
||||
actor: string;
|
||||
}
|
||||
|
||||
/** Who an alert can be handed to: people who can act on it. */
|
||||
export async function assignableUsers(): Promise<{ id: number; name: string; role: string }[]> {
|
||||
const rows = await db.select().from(users).where(inArray(users.role, ["admin", "operator"]));
|
||||
return rows.map((u) => ({ id: u.id, name: u.name ?? u.email ?? u.oidcSub, role: u.role })).sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
/** Saves an acknowledgement, replacing an earlier one on the same alert. Returns what changed, for the audit log. */
|
||||
export async function setAck(input: SetAckInput, now: Date = new Date()): Promise<{ ack: AckInfo; before: AckInfo | null }> {
|
||||
const note = input.note.trim();
|
||||
if (note.length > MAX_NOTE) throw new AckError(400, "note_too_long", `Keep the note under ${MAX_NOTE} characters.`);
|
||||
if (input.hours !== null && !(Number.isInteger(input.hours) && input.hours >= 1 && input.hours <= MAX_HOURS)) throw new AckError(400, "invalid_duration", "Choose how long it lasts, up to thirty days — or until the alert is gone.");
|
||||
|
||||
let assigneeName: string | null = null;
|
||||
if (input.assigneeId !== null) {
|
||||
const [u] = await db.select().from(users).where(eq(users.id, input.assigneeId)).limit(1);
|
||||
if (!u || (u.role !== "admin" && u.role !== "operator")) throw new AckError(400, "invalid_assignee", "An alert can be handed to an administrator or an operator.");
|
||||
assigneeName = u.name ?? u.email ?? u.oidcSub;
|
||||
}
|
||||
|
||||
const before = await loadAck(input.alertId, now);
|
||||
const until = input.hours === null ? null : new Date(now.getTime() + input.hours * 3600_000).toISOString();
|
||||
const values = { note, assigneeUserId: input.assigneeId, assigneeName, ackedBy: input.actor, ackedAt: now.toISOString(), until, lastSeenAt: now.toISOString() };
|
||||
await db.insert(alertAcks).values({ alertKey: input.alertId, ...values }).onConflictDoUpdate({ target: alertAcks.alertKey, set: values });
|
||||
return { ack: (await loadAck(input.alertId, now))!, before };
|
||||
}
|
||||
|
||||
export async function clearAck(alertId: string): Promise<AckInfo | null> {
|
||||
const before = await loadAck(alertId);
|
||||
if (!before) return null;
|
||||
await db.delete(alertAcks).where(eq(alertAcks.alertKey, alertId));
|
||||
return before;
|
||||
}
|
||||
|
||||
/**
|
||||
* Called after every collection of the alerts: the acknowledgements of alerts that are there are marked as seen, and those of alerts that
|
||||
* have been gone for half an hour are dropped. Returns how many were dropped.
|
||||
*/
|
||||
export async function reconcileAcks(presentIds: Set<string>, now: Date = new Date()): Promise<number> {
|
||||
const rows = await db.select().from(alertAcks);
|
||||
const seen = rows.filter((r) => presentIds.has(r.alertKey)).map((r) => r.id);
|
||||
if (seen.length > 0) await db.update(alertAcks).set({ lastSeenAt: now.toISOString() }).where(inArray(alertAcks.id, seen));
|
||||
const cutoff = new Date(now.getTime() - GONE_AFTER_MS).toISOString();
|
||||
const gone = rows.filter((r) => !presentIds.has(r.alertKey) && r.lastSeenAt < cutoff).map((r) => r.id);
|
||||
if (gone.length > 0) await db.delete(alertAcks).where(and(inArray(alertAcks.id, gone), lt(alertAcks.lastSeenAt, cutoff)));
|
||||
return gone.length;
|
||||
}
|
||||
|
||||
/** What still needs somebody to look: the alerts that aren't silenced and that nobody has acknowledged (or whose acknowledgement has run out). */
|
||||
export function attentionCounts(alerts: Pick<Alert, "id" | "severity" | "silenced">[], acks: Map<string, AckInfo>): { critical: number; warning: number; acknowledged: number } {
|
||||
let critical = 0;
|
||||
let warning = 0;
|
||||
let acknowledged = 0;
|
||||
for (const a of alerts) {
|
||||
if (a.silenced) continue;
|
||||
if (acks.get(a.id)?.active) {
|
||||
acknowledged++;
|
||||
continue;
|
||||
}
|
||||
if (a.severity === "critical") critical++;
|
||||
else if (a.severity === "warning") warning++;
|
||||
}
|
||||
return { critical, warning, acknowledged };
|
||||
}
|
||||
@@ -15,6 +15,9 @@ import { integrations, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
import { recordFromReport } from "./incidents.js";
|
||||
import { reconcileAcks } from "./alertAcks.js";
|
||||
import { lastAttempt as lastSelfBackup, newestSnapshot, STALE_AFTER_HOURS as SELF_BACKUP_STALE_HOURS } from "./selfBackup.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
|
||||
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
|
||||
@@ -53,6 +56,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
gitea: { label: "Gitea", link: "/gitea" },
|
||||
dockhand: { label: "Docker", link: "/docker" },
|
||||
tailscale: { label: "Tailscale", link: "/tailscale" },
|
||||
homelab: { label: "Homelab Manager", link: "/settings/backup" },
|
||||
pbs: { label: "Proxmox Backup", link: "/pbs" },
|
||||
uptimekuma: { label: "Uptime Kuma", link: "/uptime-kuma" },
|
||||
osticket: { label: "osTicket", link: "/osticket" },
|
||||
@@ -262,6 +266,29 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
}
|
||||
}),
|
||||
|
||||
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
|
||||
// stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it.
|
||||
check("Homelab Manager's own backups", async () => {
|
||||
const { selfBackup } = await getSettings();
|
||||
if (!selfBackup.enabled) return;
|
||||
const newest = newestSnapshot();
|
||||
const last = await lastSelfBackup();
|
||||
const failing = last && !last.ok ? ` The last attempt failed: ${last.error}` : "";
|
||||
if (!newest) {
|
||||
if (failing) push({ category: "backup", severity: "warning", type: "homelab", message: `Homelab Manager has no backup of its own yet.${failing}`, key: "self-backup" });
|
||||
return;
|
||||
}
|
||||
const ageHours = (Date.now() - Date.parse(newest.at)) / 3600_000;
|
||||
if (ageHours <= SELF_BACKUP_STALE_HOURS) return;
|
||||
push({
|
||||
category: "backup",
|
||||
severity: ageHours >= 24 * 7 ? "critical" : "warning",
|
||||
type: "homelab",
|
||||
message: `The newest backup of Homelab Manager's own database is ${ageHours < 72 ? `${Math.round(ageHours)} h` : `${Math.floor(ageHours / 24)} days`} old.${failing}`,
|
||||
key: "self-backup",
|
||||
});
|
||||
}),
|
||||
|
||||
check("Backup verification", async () => {
|
||||
const { problems, sourceFailures } = await collectPbsProblems({ skipSilenced: false });
|
||||
for (const p of problems) {
|
||||
@@ -475,6 +502,8 @@ export async function getAlerts(force: boolean): Promise<AlertsReport & { cached
|
||||
inFlight ??= collectAlerts()
|
||||
.then((report) => {
|
||||
cache = { at: Date.now(), report };
|
||||
// Remembered for the incident record and the acknowledgements — in the background, and never allowed to hold up or break the answer.
|
||||
void recordFromReport(report).then(() => reconcileAcks(new Set(report.alerts.map((a) => a.id)))).catch((err) => console.error("[alerts] couldn't update the incident record:", err));
|
||||
return report;
|
||||
})
|
||||
.finally(() => {
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
/**
|
||||
* Incidents: each time an alert is raised and later goes away is one incident, with the moment it began and the moment it ended — so
|
||||
* "what happened on Tuesday night?" has an answer after the alert itself is long gone. Around each one the page puts together everything the
|
||||
* app knows about that time: who acknowledged it and why, the maintenance windows, what was changed on the server, the update runs, the
|
||||
* other alerts that came with it, and the notes people wrote.
|
||||
*
|
||||
* The record is made from the alerts as they are collected (and a check every five minutes so it doesn't depend on anybody looking), by
|
||||
* comparing what is there now with what was open. An alert that isn't there is only called resolved after it has been missing for a couple
|
||||
* of minutes in a row — much longer when a check couldn't run, since "not there" then may only mean "couldn't look" — and never in the first
|
||||
* minutes after the app starts, when server alerts are held back on purpose.
|
||||
*/
|
||||
import { and, asc, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { alertIncidents, auditLog, incidentNotes, maintenanceWindows, updateRuns } from "../db/schema.js";
|
||||
import type { Alert } from "./alertTypes.js";
|
||||
import { describeAudit, auditTimeToIso, type ActivityTone } from "./serverActivity.js";
|
||||
import { startupGraceRemainingMs } from "./healthMonitor.js";
|
||||
import { loadAck, type AckInfo } from "./alertAcks.js";
|
||||
|
||||
export type IncidentRow = typeof alertIncidents.$inferSelect;
|
||||
|
||||
/** How long an alert must be missing before the incident is called over. */
|
||||
export const MISSING_GRACE_MS = 2 * 60_000;
|
||||
/** The same, when a check couldn't run on the last look: not there may then only mean couldn't look. */
|
||||
export const MISSING_GRACE_BLIND_MS = 30 * 60_000;
|
||||
/** Resolved incidents are kept this long. */
|
||||
export const KEEP_DAYS = 400;
|
||||
/** Other alerts raised this close to an incident, about the same thing, are shown with it. */
|
||||
export const RELATED_WINDOW_MS = 15 * 60_000;
|
||||
|
||||
const RANK: Record<string, number> = { info: 0, warning: 1, critical: 2 };
|
||||
|
||||
export interface SnapshotInput {
|
||||
alerts: Alert[];
|
||||
/** The checks that couldn't run on this look. */
|
||||
blind: number;
|
||||
generatedAt?: string;
|
||||
}
|
||||
|
||||
export interface SnapshotResult {
|
||||
opened: number[];
|
||||
resolved: number[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Brings the incidents in line with the alerts as they are now. Information-only alerts aren't incidents. Safe to call as often as likes:
|
||||
* the same look twice changes nothing.
|
||||
*/
|
||||
export async function recordAlertSnapshot(input: SnapshotInput, now: Date = new Date(), options: { inGrace?: boolean } = {}): Promise<SnapshotResult> {
|
||||
const iso = now.toISOString();
|
||||
const present = new Map(input.alerts.filter((a) => a.severity !== "info").map((a) => [a.id, a]));
|
||||
const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt));
|
||||
const openByKey = new Map(open.map((i) => [i.alertKey, i]));
|
||||
const result: SnapshotResult = { opened: [], resolved: [] };
|
||||
|
||||
for (const [key, a] of present) {
|
||||
const existing = openByKey.get(key);
|
||||
if (existing) {
|
||||
const peak = (RANK[a.severity] ?? 0) > (RANK[existing.peakSeverity] ?? 0) ? a.severity : existing.peakSeverity;
|
||||
await db
|
||||
.update(alertIncidents)
|
||||
.set({ severity: a.severity, peakSeverity: peak, message: a.message, causedBy: a.causedBy?.name ?? null, lastSeenAt: iso, missingSince: null })
|
||||
.where(eq(alertIncidents.id, existing.id));
|
||||
} else {
|
||||
const [row] = await db
|
||||
.insert(alertIncidents)
|
||||
.values({
|
||||
alertKey: key,
|
||||
severity: a.severity,
|
||||
peakSeverity: a.severity,
|
||||
category: a.category,
|
||||
source: a.source,
|
||||
message: a.message,
|
||||
link: a.link,
|
||||
targetType: a.target?.type ?? null,
|
||||
targetId: a.target?.id ?? null,
|
||||
targetName: a.target?.name ?? null,
|
||||
causedBy: a.causedBy?.name ?? null,
|
||||
openedAt: iso,
|
||||
lastSeenAt: iso,
|
||||
})
|
||||
.returning({ id: alertIncidents.id });
|
||||
result.opened.push(row.id);
|
||||
}
|
||||
}
|
||||
|
||||
// What was open and isn't there now. Nothing is called over while the app is still settling after a start, when server alerts are held back.
|
||||
const grace = input.blind > 0 ? MISSING_GRACE_BLIND_MS : MISSING_GRACE_MS;
|
||||
for (const inc of open) {
|
||||
if (present.has(inc.alertKey)) continue;
|
||||
if (options.inGrace) continue;
|
||||
if (inc.missingSince === null) {
|
||||
await db.update(alertIncidents).set({ missingSince: iso }).where(eq(alertIncidents.id, inc.id));
|
||||
} else if (now.getTime() - Date.parse(inc.missingSince) >= grace) {
|
||||
// Over since the first look it was missing, not since this one.
|
||||
await db.update(alertIncidents).set({ resolvedAt: inc.missingSince, missingSince: null }).where(eq(alertIncidents.id, inc.id));
|
||||
result.resolved.push(inc.id);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Called by the collection of the alerts: records what it found, and never lets a problem here stop the alerts from being served. */
|
||||
export async function recordFromReport(report: { alerts: Alert[]; couldntCheck: unknown[] }, now: Date = new Date()): Promise<void> {
|
||||
try {
|
||||
await recordAlertSnapshot({ alerts: report.alerts, blind: report.couldntCheck.length }, now, { inGrace: startupGraceRemainingMs(now.getTime()) > 0 });
|
||||
} catch (err) {
|
||||
console.error("[incidents] couldn't record the alerts:", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolved incidents older than the keeping time are removed (with their notes). */
|
||||
export async function pruneIncidents(now: Date = new Date()): Promise<number> {
|
||||
const cutoff = new Date(now.getTime() - KEEP_DAYS * 86_400_000).toISOString();
|
||||
const old = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(and(isNotNull(alertIncidents.resolvedAt), lte(alertIncidents.resolvedAt, cutoff)));
|
||||
if (old.length === 0) return 0;
|
||||
await db.delete(alertIncidents).where(inArray(alertIncidents.id, old.map((o) => o.id)));
|
||||
return old.length;
|
||||
}
|
||||
|
||||
// ─── Reading them ───────────────────────────────────────────────────────────
|
||||
|
||||
export interface IncidentSummary {
|
||||
id: number;
|
||||
alertKey: string;
|
||||
severity: string;
|
||||
peakSeverity: string;
|
||||
category: string;
|
||||
source: string;
|
||||
message: string;
|
||||
link: string | null;
|
||||
target: { type: string; id: number; name: string } | null;
|
||||
causedBy: string | null;
|
||||
openedAt: string;
|
||||
resolvedAt: string | null;
|
||||
/** In milliseconds: until it was resolved, or until now while it is open. */
|
||||
durationMs: number;
|
||||
open: boolean;
|
||||
notes: number;
|
||||
}
|
||||
|
||||
const summarize = (r: IncidentRow, notes: number, now: Date): IncidentSummary => ({
|
||||
id: r.id,
|
||||
alertKey: r.alertKey,
|
||||
severity: r.severity,
|
||||
peakSeverity: r.peakSeverity,
|
||||
category: r.category,
|
||||
source: r.source,
|
||||
message: r.message,
|
||||
link: r.link,
|
||||
target: r.targetType && r.targetId !== null ? { type: r.targetType, id: r.targetId, name: r.targetName ?? `${r.targetType} #${r.targetId}` } : null,
|
||||
causedBy: r.causedBy,
|
||||
openedAt: r.openedAt,
|
||||
resolvedAt: r.resolvedAt,
|
||||
durationMs: Math.max(0, (r.resolvedAt ? Date.parse(r.resolvedAt) : now.getTime()) - Date.parse(r.openedAt)),
|
||||
open: r.resolvedAt === null,
|
||||
notes,
|
||||
});
|
||||
|
||||
/** Open incidents first (newest first), then the resolved ones, newest first. */
|
||||
export async function listIncidents(options: { limit?: number; now?: Date } = {}): Promise<IncidentSummary[]> {
|
||||
const now = options.now ?? new Date();
|
||||
const limit = options.limit ?? 300;
|
||||
const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt));
|
||||
const closed = await db.select().from(alertIncidents).where(isNotNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt)).limit(limit);
|
||||
const rows = [...open, ...closed];
|
||||
const noteRows = rows.length > 0 ? await db.select({ incidentId: incidentNotes.incidentId }).from(incidentNotes).where(inArray(incidentNotes.incidentId, rows.map((r) => r.id))) : [];
|
||||
const counts = new Map<number, number>();
|
||||
for (const n of noteRows) counts.set(n.incidentId, (counts.get(n.incidentId) ?? 0) + 1);
|
||||
return rows.map((r) => summarize(r, counts.get(r.id) ?? 0, now));
|
||||
}
|
||||
|
||||
/** The open incident for each alert id, so the Alerts page can link to it. */
|
||||
export async function openIncidentIds(alertKeys: string[]): Promise<Map<string, number>> {
|
||||
if (alertKeys.length === 0) return new Map();
|
||||
const rows = await db.select({ id: alertIncidents.id, key: alertIncidents.alertKey }).from(alertIncidents).where(and(isNull(alertIncidents.resolvedAt), inArray(alertIncidents.alertKey, alertKeys)));
|
||||
return new Map(rows.map((r) => [r.key, r.id]));
|
||||
}
|
||||
|
||||
export type TimelineKind = "alert" | "ack" | "maintenance" | "change" | "health" | "run" | "related" | "note";
|
||||
|
||||
export interface TimelineEvent {
|
||||
/** ISO. */
|
||||
at: string;
|
||||
kind: TimelineKind;
|
||||
tone: ActivityTone;
|
||||
title: string;
|
||||
detail: string | null;
|
||||
actor: string | null;
|
||||
/** For a related alert or a note: what it points at. */
|
||||
incidentId?: number;
|
||||
noteId?: number;
|
||||
}
|
||||
|
||||
export interface IncidentDetail {
|
||||
incident: IncidentSummary;
|
||||
/** The acknowledgement in force now, if the alert is still open. */
|
||||
ack: AckInfo | null;
|
||||
timeline: TimelineEvent[];
|
||||
notes: { id: number; author: string; body: string; at: string }[];
|
||||
}
|
||||
|
||||
function parse(detail: string | null): Record<string, unknown> {
|
||||
if (!detail) return {};
|
||||
try {
|
||||
const v = JSON.parse(detail);
|
||||
return v && typeof v === "object" ? (v as Record<string, unknown>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
const words = (ms: number) => {
|
||||
const m = Math.round(ms / 60_000);
|
||||
if (m < 90) return `${Math.max(1, m)} min`;
|
||||
const h = ms / 3600_000;
|
||||
return h < 48 ? `${Math.round(h * 10) / 10} h` : `${Math.round(h / 24)} days`;
|
||||
};
|
||||
|
||||
/** Everything the app knows about one incident, in the order it happened. */
|
||||
export async function getIncident(id: number, now: Date = new Date()): Promise<IncidentDetail | null> {
|
||||
const [row] = await db.select().from(alertIncidents).where(eq(alertIncidents.id, id)).limit(1);
|
||||
if (!row) return null;
|
||||
const notes = await db.select().from(incidentNotes).where(eq(incidentNotes.incidentId, id)).orderBy(asc(incidentNotes.createdAt));
|
||||
const incident = summarize(row, notes.length, now);
|
||||
|
||||
const from = Date.parse(row.openedAt);
|
||||
const to = row.resolvedAt ? Date.parse(row.resolvedAt) : now.getTime();
|
||||
const fromIso = new Date(from - 30 * 60_000).toISOString();
|
||||
const toIso = new Date(to + 15 * 60_000).toISOString();
|
||||
const events: TimelineEvent[] = [];
|
||||
|
||||
events.push({ at: row.openedAt, kind: "alert", tone: row.peakSeverity === "critical" ? "bad" : "info", title: `Alert raised (${row.severity})`, detail: row.message, actor: null });
|
||||
if (row.peakSeverity !== row.severity) {
|
||||
events.push({ at: row.resolvedAt ?? row.lastSeenAt, kind: "alert", tone: "neutral", title: `At its worst it was ${row.peakSeverity}`, detail: null, actor: null });
|
||||
}
|
||||
if (row.causedBy) events.push({ at: row.openedAt, kind: "alert", tone: "info", title: `Held back: caused by ${row.causedBy}`, detail: "Folded under the alert for the machine it runs on.", actor: null });
|
||||
if (row.resolvedAt) events.push({ at: row.resolvedAt, kind: "alert", tone: "good", title: "Resolved", detail: `It lasted ${words(to - from)}.`, actor: null });
|
||||
|
||||
// Acknowledgements and hand-overs of this alert, from the audit log.
|
||||
const audits = await db.select().from(auditLog).where(and(gte(auditLog.createdAt, fromIso.slice(0, 19).replace("T", " ")), lte(auditLog.createdAt, toIso.slice(0, 19).replace("T", " ")))).orderBy(asc(auditLog.createdAt));
|
||||
for (const a of audits) {
|
||||
const d = parse(a.detail);
|
||||
if (a.category === "alert") {
|
||||
if (d.alertId !== row.alertKey) continue;
|
||||
const described = describeAudit(a.category, a.action, d);
|
||||
events.push({ at: auditTimeToIso(a.createdAt), kind: "ack", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel });
|
||||
continue;
|
||||
}
|
||||
// Things done to, or noticed about, the thing the alert is about.
|
||||
if (row.targetType && row.targetId !== null && a.targetType === row.targetType && a.targetId === String(row.targetId) && a.category !== "maintenance") {
|
||||
const described = describeAudit(a.category, a.action, d);
|
||||
events.push({ at: auditTimeToIso(a.createdAt), kind: described.kind === "health" ? "health" : "change", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel });
|
||||
}
|
||||
}
|
||||
|
||||
if (row.targetType && row.targetId !== null) {
|
||||
const windows = await db
|
||||
.select()
|
||||
.from(maintenanceWindows)
|
||||
.where(and(eq(maintenanceWindows.targetType, row.targetType as "server"), eq(maintenanceWindows.targetId, row.targetId), lte(maintenanceWindows.startedAt, new Date(to).toISOString()), gte(maintenanceWindows.endsAt, new Date(from).toISOString())));
|
||||
for (const w of windows) {
|
||||
if (w.endsAt <= w.startedAt) continue; // withdrawn before it began
|
||||
events.push({ at: w.startedAt, kind: "maintenance", tone: "info", title: "Maintenance began — alerts silenced", detail: w.reason, actor: w.createdBy });
|
||||
if (w.endsAt <= toIso) events.push({ at: w.endsAt, kind: "maintenance", tone: "neutral", title: "Maintenance ended", detail: null, actor: null });
|
||||
}
|
||||
if (row.targetType === "server") {
|
||||
const runs = await db.select().from(updateRuns).where(and(eq(updateRuns.serverId, row.targetId), lte(updateRuns.startedAt, new Date(to).toISOString()), or(isNull(updateRuns.finishedAt), gte(updateRuns.finishedAt, new Date(from).toISOString()))!));
|
||||
for (const r of runs) {
|
||||
events.push({ at: r.startedAt, kind: "run", tone: "info", title: `Update run started: ${r.templateName}`, detail: `by ${r.startedBy ?? "someone"}`, actor: r.startedBy });
|
||||
if (r.finishedAt) events.push({ at: r.finishedAt, kind: "run", tone: r.status === "done" ? "good" : "bad", title: r.status === "done" ? "Update run finished" : r.status === "cancelled" ? "Update run stopped being followed" : "Update run didn't finish", detail: r.message, actor: null });
|
||||
}
|
||||
}
|
||||
|
||||
// Other alerts about the same thing that began around the same time.
|
||||
const related = await db
|
||||
.select()
|
||||
.from(alertIncidents)
|
||||
.where(and(eq(alertIncidents.targetType, row.targetType), eq(alertIncidents.targetId, row.targetId), ne(alertIncidents.id, row.id), gte(alertIncidents.openedAt, new Date(from - RELATED_WINDOW_MS).toISOString()), lte(alertIncidents.openedAt, new Date(Math.max(to, from) + RELATED_WINDOW_MS).toISOString())));
|
||||
for (const r of related) events.push({ at: r.openedAt, kind: "related", tone: "info", title: `Another alert was raised: ${r.category} (${r.severity})`, detail: r.message, actor: null, incidentId: r.id });
|
||||
}
|
||||
|
||||
for (const n of notes) events.push({ at: n.createdAt, kind: "note", tone: "neutral", title: `${n.author} wrote a note`, detail: n.body, actor: n.author, noteId: n.id });
|
||||
|
||||
events.sort((a, b) => a.at.localeCompare(b.at));
|
||||
return { incident, ack: row.resolvedAt ? null : await loadAck(row.alertKey, now), timeline: events, notes: notes.map((n) => ({ id: n.id, author: n.author, body: n.body, at: n.createdAt })) };
|
||||
}
|
||||
|
||||
export const MAX_NOTE = 4000;
|
||||
|
||||
export async function addNote(incidentId: number, author: string, body: string, now: Date = new Date()): Promise<{ id: number } | null> {
|
||||
const [inc] = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(eq(alertIncidents.id, incidentId)).limit(1);
|
||||
if (!inc) return null;
|
||||
const [row] = await db.insert(incidentNotes).values({ incidentId, author, body: body.trim(), createdAt: now.toISOString() }).returning({ id: incidentNotes.id });
|
||||
return row;
|
||||
}
|
||||
|
||||
export async function removeNote(incidentId: number, noteId: number, actor: { name: string; isAdmin: boolean }): Promise<"removed" | "not_found" | "forbidden"> {
|
||||
const [n] = await db.select().from(incidentNotes).where(and(eq(incidentNotes.id, noteId), eq(incidentNotes.incidentId, incidentId))).limit(1);
|
||||
if (!n) return "not_found";
|
||||
if (n.author !== actor.name && !actor.isAdmin) return "forbidden";
|
||||
await db.delete(incidentNotes).where(eq(incidentNotes.id, noteId));
|
||||
return "removed";
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
/** A look at the alerts every five minutes, so incidents are recorded whether or not anybody has the page open. Once a day it tidies up old ones. */
|
||||
export function initIncidentRecorder(lookAtAlerts: () => Promise<unknown>): void {
|
||||
if (timer) return;
|
||||
let lastPrune = 0;
|
||||
const tick = async () => {
|
||||
try {
|
||||
await lookAtAlerts();
|
||||
if (Date.now() - lastPrune > 24 * 3600_000) {
|
||||
lastPrune = Date.now();
|
||||
await pruneIncidents();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[incidents] the five-minute look failed:", err);
|
||||
}
|
||||
};
|
||||
setTimeout(() => void tick(), 60_000).unref();
|
||||
timer = setInterval(() => void tick(), 5 * 60_000);
|
||||
timer.unref();
|
||||
}
|
||||
@@ -13,6 +13,9 @@ import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
|
||||
import { parseThresholds } from "./serverThresholds.js";
|
||||
import { getCachedBackupCoverage } from "./backupCoverage.js";
|
||||
import { lastAttempt as lastSelfBackup, listSnapshots } from "./selfBackup.js";
|
||||
import { loadAcks, attentionCounts } from "./alertAcks.js";
|
||||
import { listIncidents } from "./incidents.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
@@ -115,6 +118,18 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
console.error("[metrics] couldn't read the uptime:", err);
|
||||
}
|
||||
|
||||
// ── Homelab Manager's own backups ──
|
||||
try {
|
||||
const snapshots = listSnapshots();
|
||||
out.add("homelab_selfbackup_snapshots", "gauge", "How many backups of Homelab Manager's own database are kept.", snapshots.length);
|
||||
const newest = snapshots.find((s) => s.kind === "scheduled" || s.kind === "manual");
|
||||
if (newest) out.add("homelab_selfbackup_last_success_timestamp_seconds", "gauge", "When the newest backup of Homelab Manager's own database was made, as a Unix time.", Math.floor(Date.parse(newest.at) / 1000));
|
||||
const last = await lastSelfBackup();
|
||||
if (last) out.add("homelab_selfbackup_last_attempt_ok", "gauge", "1 if the last attempt to back up Homelab Manager's own database worked, 0 if it failed.", last.ok ? 1 : 0);
|
||||
} catch (err) {
|
||||
console.error("[metrics] couldn't read the backups folder:", err);
|
||||
}
|
||||
|
||||
// ── Alerts ──
|
||||
try {
|
||||
const report = await getAlerts(false);
|
||||
@@ -127,6 +142,9 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
}
|
||||
}
|
||||
for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity });
|
||||
// Somebody has said they are dealing with these (still raised, and still counted above).
|
||||
out.add("homelab_alerts_acknowledged", "gauge", "Alerts somebody has acknowledged and that are not silenced by maintenance.", attentionCounts(report.alerts, await loadAcks(now)).acknowledged);
|
||||
out.add("homelab_incidents_open", "gauge", "Incidents that have begun and not yet ended.", (await listIncidents({ limit: 1, now })).filter((i) => i.open).length);
|
||||
out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced);
|
||||
out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped);
|
||||
} catch (err) {
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
/**
|
||||
* Backups of Homelab Manager itself. Everything it knows is one SQLite file, so a backup is a consistent copy of that file (`VACUUM INTO`,
|
||||
* which is safe while the app is working) kept in the data folder's `backups/`: taken every day, the newest few kept, each one checked as
|
||||
* it is made, and downloadable, uploadable and restorable from Settings → Backup.
|
||||
*
|
||||
* What a backup is not: it does not hold `CREDENTIALS_ENCRYPTION_KEY`, which decrypts the credentials stored in it, and it isn't off the
|
||||
* machine until someone copies it there. Both are said where backups are shown.
|
||||
*
|
||||
* A restore can't swap a database that is open, so it is staged (checked, then copied beside the database) and takes effect when the app
|
||||
* next starts — see db/applyRestore.ts, which does it before anything opens the file.
|
||||
*/
|
||||
import { createClient } from "@libsql/client";
|
||||
import { copyFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, openSync, readdirSync, readFileSync, readSync, closeSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { client, dbPath } from "../db/client.js";
|
||||
import { PENDING_FILE, PENDING_NOTE, RESULT_NOTE, type RestoreResult } from "../db/restoreFiles.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export type SnapshotKind = "scheduled" | "manual" | "pre-restore" | "uploaded";
|
||||
|
||||
export interface SnapshotInfo {
|
||||
name: string;
|
||||
kind: SnapshotKind;
|
||||
sizeBytes: number;
|
||||
/** ISO — when the file was written. */
|
||||
at: string;
|
||||
}
|
||||
|
||||
export interface SnapshotCheck {
|
||||
ok: boolean;
|
||||
/** SQLite's own verdict ("ok", or what it found). */
|
||||
integrity: string | null;
|
||||
/** How many schema changes it has had; goes up with each release that changes the database. */
|
||||
migrations: number | null;
|
||||
/** How many this version of the app knows. */
|
||||
appMigrations: number;
|
||||
/** Made by a newer version of the app than this one: it can't be restored here. */
|
||||
newerThanApp: boolean;
|
||||
counts: { users: number; servers: number; integrations: number } | null;
|
||||
/** Why it isn't usable, when it isn't. */
|
||||
problem: string | null;
|
||||
}
|
||||
|
||||
const NAME = /^homelab-manager-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}(?:-(manual|pre-restore|uploaded))?(?:-\d+)?\.sqlite$/;
|
||||
export const isSnapshotName = (name: string) => NAME.test(name);
|
||||
|
||||
const dataDir = () => dirname(dbPath);
|
||||
export const backupDir = () => join(dataDir(), "backups");
|
||||
|
||||
const stamp = (d: Date) => d.toISOString().slice(0, 19).replace(/[:T]/g, "-");
|
||||
|
||||
function kindOf(name: string): SnapshotKind {
|
||||
const m = NAME.exec(name);
|
||||
return (m?.[1] as SnapshotKind | undefined) ?? "scheduled";
|
||||
}
|
||||
|
||||
/** How many schema changes this version of the app has (the entries in its migration journal). */
|
||||
export function appMigrationCount(): number {
|
||||
try {
|
||||
const journal = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "drizzle", "meta", "_journal.json"), "utf8")) as { entries?: unknown[] };
|
||||
return journal.entries?.length ?? 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function hasSqliteHeader(file: string): boolean {
|
||||
try {
|
||||
const fd = openSync(file, "r");
|
||||
try {
|
||||
const buf = Buffer.alloc(16);
|
||||
readSync(fd, buf, 0, 16, 0);
|
||||
return buf.toString("latin1") === "SQLite format 3\0";
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** A scratch folder for looking at a copy of a backup, and the way to be rid of it (best effort: some systems hold an opened database until the process ends). */
|
||||
function scratch(): { file: string; done: () => void } {
|
||||
const folder = mkdtempSync(join(tmpdir(), "hlm-backup-"));
|
||||
return {
|
||||
file: join(folder, "db.sqlite"),
|
||||
done: () => {
|
||||
try {
|
||||
rmSync(folder, { recursive: true, force: true });
|
||||
} catch {
|
||||
// left for the system's own tidying of its temporary folder
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Looks at a backup: does SQLite think it is sound, is it one of ours, and is it from this version or an older one? It looks at a copy in a
|
||||
* temporary folder, never at the file itself, so a backup in the backups folder is never held open — it can always be deleted, anywhere.
|
||||
*/
|
||||
export async function checkSnapshotFile(file: string): Promise<SnapshotCheck> {
|
||||
if (!existsSync(file)) return checkOpened(file);
|
||||
if (!hasSqliteHeader(file)) return checkOpened(file);
|
||||
const tmp = scratch();
|
||||
try {
|
||||
copyFileSync(file, tmp.file);
|
||||
return await checkOpened(tmp.file);
|
||||
} finally {
|
||||
tmp.done();
|
||||
}
|
||||
}
|
||||
|
||||
/** The check itself, on a file nobody else cares about the handle of. */
|
||||
async function checkOpened(file: string): Promise<SnapshotCheck> {
|
||||
const appMigrations = appMigrationCount();
|
||||
const fail = (problem: string, extra: Partial<SnapshotCheck> = {}): SnapshotCheck => ({ ok: false, integrity: null, migrations: null, appMigrations, newerThanApp: false, counts: null, problem, ...extra });
|
||||
if (!existsSync(file)) return fail("The file isn't there.");
|
||||
if (!hasSqliteHeader(file)) return fail("That isn't a database file.");
|
||||
const c = createClient({ url: `file:${file}` });
|
||||
try {
|
||||
const one = async (sql: string) => Object.values((await c.execute(sql)).rows[0] ?? {})[0];
|
||||
const integrity = String(await one("PRAGMA integrity_check"));
|
||||
const tables = new Set((await c.execute("select name from sqlite_master where type = 'table'")).rows.map((r) => String(r.name)));
|
||||
for (const needed of ["users", "servers", "integrations", "settings", "__drizzle_migrations"]) {
|
||||
if (!tables.has(needed)) return fail(`It isn't a Homelab Manager database (no "${needed}" table).`, { integrity });
|
||||
}
|
||||
const migrations = Number(await one("select count(*) from __drizzle_migrations"));
|
||||
const counts = { users: Number(await one("select count(*) from users")), servers: Number(await one("select count(*) from servers")), integrations: Number(await one("select count(*) from integrations")) };
|
||||
const newerThanApp = appMigrations > 0 && migrations > appMigrations;
|
||||
if (integrity !== "ok") return fail(`SQLite found problems in it: ${integrity}`, { integrity, migrations, counts, newerThanApp });
|
||||
if (newerThanApp) return fail("It was made by a newer version of Homelab Manager than this one, and an older version can't use it. Update first.", { integrity, migrations, counts, newerThanApp });
|
||||
return { ok: true, integrity, migrations, appMigrations, newerThanApp: false, counts, problem: null };
|
||||
} catch (err) {
|
||||
return fail(`It couldn't be read as a database: ${err instanceof Error ? err.message : String(err)}`);
|
||||
} finally {
|
||||
c.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** A name for a new file that isn't taken. */
|
||||
function freshName(kind: SnapshotKind, now: Date): string {
|
||||
const base = `homelab-manager-${stamp(now)}${kind === "scheduled" ? "" : `-${kind}`}`;
|
||||
let name = `${base}.sqlite`;
|
||||
for (let n = 2; existsSync(join(backupDir(), name)); n++) name = `${base}-${n}.sqlite`;
|
||||
return name;
|
||||
}
|
||||
|
||||
export interface LastAttempt {
|
||||
at: string;
|
||||
ok: boolean;
|
||||
kind: SnapshotKind;
|
||||
error: string | null;
|
||||
name: string | null;
|
||||
}
|
||||
const ATTEMPT_FLAG = "selfBackupLastAttempt";
|
||||
|
||||
export async function lastAttempt(): Promise<LastAttempt | null> {
|
||||
try {
|
||||
const v = await getInternalFlag(ATTEMPT_FLAG);
|
||||
return v ? (JSON.parse(v) as LastAttempt) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes a consistent copy of the database into the backups folder and checks it. Throws, saying why, if it can't — and records the attempt either way. */
|
||||
export async function createSnapshot(kind: SnapshotKind = "manual", now: Date = new Date()): Promise<{ info: SnapshotInfo; check: SnapshotCheck }> {
|
||||
const tmp = scratch();
|
||||
try {
|
||||
mkdirSync(backupDir(), { recursive: true });
|
||||
const name = freshName(kind, now);
|
||||
const file = join(backupDir(), name);
|
||||
// Made and checked in a temporary folder first, and only then put in the backups folder: what is there is always whole and sound.
|
||||
await client.execute(`VACUUM INTO '${tmp.file.replace(/'/g, "''")}'`);
|
||||
const check = await checkOpened(tmp.file);
|
||||
if (!check.ok) throw new Error(`The copy didn't check out, so it was thrown away: ${check.problem}`);
|
||||
copyFileSync(tmp.file, file);
|
||||
try {
|
||||
chmodSync(file, 0o600);
|
||||
} catch {
|
||||
// not every file system has modes
|
||||
}
|
||||
const info: SnapshotInfo = { name, kind, sizeBytes: statSync(file).size, at: now.toISOString() };
|
||||
await setInternalFlag(ATTEMPT_FLAG, JSON.stringify({ at: now.toISOString(), ok: true, kind, error: null, name } satisfies LastAttempt));
|
||||
return { info, check };
|
||||
} catch (err) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
await setInternalFlag(ATTEMPT_FLAG, JSON.stringify({ at: now.toISOString(), ok: false, kind, error, name: null } satisfies LastAttempt));
|
||||
throw err;
|
||||
} finally {
|
||||
tmp.done();
|
||||
}
|
||||
}
|
||||
|
||||
export function listSnapshots(): SnapshotInfo[] {
|
||||
if (!existsSync(backupDir())) return [];
|
||||
return readdirSync(backupDir())
|
||||
.filter(isSnapshotName)
|
||||
.map((name) => {
|
||||
const st = statSync(join(backupDir(), name));
|
||||
return { name, kind: kindOf(name), sizeBytes: st.size, at: st.mtime.toISOString() };
|
||||
})
|
||||
.sort((a, b) => b.at.localeCompare(a.at) || b.name.localeCompare(a.name));
|
||||
}
|
||||
|
||||
/** What is kept: the newest `keep` scheduled backups, and a few of each other kind (a restore's "before" copy is worth keeping longer than a day). */
|
||||
export const KEEP_OTHER: Record<Exclude<SnapshotKind, "scheduled">, number> = { manual: 10, "pre-restore": 3, uploaded: 3 };
|
||||
|
||||
export function pruneSnapshots(keep: number): string[] {
|
||||
const removed: string[] = [];
|
||||
const all = listSnapshots();
|
||||
const limitFor = (kind: SnapshotKind) => (kind === "scheduled" ? Math.max(1, keep) : KEEP_OTHER[kind]);
|
||||
for (const kind of ["scheduled", "manual", "pre-restore", "uploaded"] as const) {
|
||||
for (const s of all.filter((x) => x.kind === kind).slice(limitFor(kind))) {
|
||||
rmSync(join(backupDir(), s.name), { force: true });
|
||||
removed.push(s.name);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
export function snapshotPath(name: string): string | null {
|
||||
if (!isSnapshotName(name)) return null;
|
||||
const file = join(backupDir(), name);
|
||||
return existsSync(file) ? file : null;
|
||||
}
|
||||
|
||||
export function deleteSnapshot(name: string): boolean {
|
||||
const file = snapshotPath(name);
|
||||
if (!file) return false;
|
||||
rmSync(file, { force: true });
|
||||
return true;
|
||||
}
|
||||
|
||||
/** A file somebody uploaded, kept as a backup if it checks out (and thrown away if not). */
|
||||
export async function importUploaded(bytes: Buffer, now: Date = new Date()): Promise<{ info: SnapshotInfo; check: SnapshotCheck }> {
|
||||
const tmp = scratch();
|
||||
try {
|
||||
writeFileSync(tmp.file, bytes);
|
||||
const check = await checkOpened(tmp.file);
|
||||
if (!check.ok) throw new Error(check.problem ?? "That file isn't usable.");
|
||||
mkdirSync(backupDir(), { recursive: true });
|
||||
const name = freshName("uploaded", now);
|
||||
const file = join(backupDir(), name);
|
||||
copyFileSync(tmp.file, file);
|
||||
try {
|
||||
chmodSync(file, 0o600);
|
||||
} catch {
|
||||
// not every file system has modes
|
||||
}
|
||||
return { info: { name, kind: "uploaded", sizeBytes: statSync(file).size, at: now.toISOString() }, check };
|
||||
} finally {
|
||||
tmp.done();
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Restoring ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface StagedRestore {
|
||||
from: string;
|
||||
stagedAt: string;
|
||||
by: string | null;
|
||||
}
|
||||
|
||||
export function stagedRestore(): StagedRestore | null {
|
||||
try {
|
||||
if (!existsSync(join(dataDir(), PENDING_FILE))) return null;
|
||||
const note = JSON.parse(readFileSync(join(dataDir(), PENDING_NOTE), "utf8")) as { from?: string; stagedAt?: string; by?: string | null };
|
||||
return { from: note.from ?? "a backup", stagedAt: note.stagedAt ?? "", by: note.by ?? null };
|
||||
} catch {
|
||||
return existsSync(join(dataDir(), PENDING_FILE)) ? { from: "a backup", stagedAt: "", by: null } : null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Checks a backup and stages it to replace the database the next time the app starts. Throws, saying why, if it can't be used. */
|
||||
export async function stageRestore(name: string, by: string | null, now: Date = new Date()): Promise<SnapshotCheck> {
|
||||
const file = snapshotPath(name);
|
||||
if (!file) throw new Error("There is no backup with that name.");
|
||||
const check = await checkSnapshotFile(file);
|
||||
if (!check.ok) throw new Error(check.problem ?? "That backup isn't usable.");
|
||||
const pending = join(dataDir(), PENDING_FILE);
|
||||
const tmp = `${pending}.copying`;
|
||||
copyFileSync(file, tmp);
|
||||
renameSync(tmp, pending); // whole, or not there at all: the start-up step never sees half a file
|
||||
try {
|
||||
chmodSync(pending, 0o600);
|
||||
} catch {
|
||||
// not every file system has modes
|
||||
}
|
||||
writeFileSync(join(dataDir(), PENDING_NOTE), JSON.stringify({ from: name, stagedAt: now.toISOString(), by }));
|
||||
return check;
|
||||
}
|
||||
|
||||
export function cancelStagedRestore(): boolean {
|
||||
const had = existsSync(join(dataDir(), PENDING_FILE));
|
||||
rmSync(join(dataDir(), PENDING_FILE), { force: true });
|
||||
rmSync(join(dataDir(), PENDING_NOTE), { force: true });
|
||||
return had;
|
||||
}
|
||||
|
||||
/** What the last restore did, once, for the page to say. */
|
||||
export function lastRestore(): RestoreResult | null {
|
||||
try {
|
||||
return JSON.parse(readFileSync(join(dataDir(), RESULT_NOTE), "utf8")) as RestoreResult;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function dismissLastRestore(): void {
|
||||
rmSync(join(dataDir(), RESULT_NOTE), { force: true });
|
||||
}
|
||||
|
||||
// ─── How current the backups are ────────────────────────────────────────────
|
||||
|
||||
/** Backups older than this are an alert: a day's slack on top of the daily schedule. */
|
||||
export const STALE_AFTER_HOURS = 50;
|
||||
|
||||
export function newestSnapshot(): SnapshotInfo | null {
|
||||
return listSnapshots().find((s) => s.kind === "scheduled" || s.kind === "manual") ?? null;
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import schedule from "node-schedule";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { createSnapshot, lastAttempt, newestSnapshot, pruneSnapshots } from "./selfBackup.js";
|
||||
import { notify } from "./notify.js";
|
||||
|
||||
let currentJob: schedule.Job | null = null;
|
||||
|
||||
/** The cron rule for "every day at HH:MM". */
|
||||
export function cronForBackup(time: string): string {
|
||||
const [h, m] = time.split(":").map(Number);
|
||||
return `${Number.isFinite(m) ? m : 30} ${Number.isFinite(h) ? h : 3} * * *`;
|
||||
}
|
||||
|
||||
/** One scheduled backup: take it, then drop the oldest beyond how many are kept. Says so through the channels if it fails. */
|
||||
export async function runScheduledBackup(now: Date = new Date()): Promise<{ name: string } | null> {
|
||||
const { selfBackup, notifications } = await getSettings();
|
||||
try {
|
||||
const { info } = await createSnapshot("scheduled", now);
|
||||
pruneSnapshots(selfBackup.keep);
|
||||
return { name: info.name };
|
||||
} catch (err) {
|
||||
console.error("[selfBackup] the scheduled backup failed:", err instanceof Error ? err.message : err);
|
||||
if (notifications.healthAlerts !== false) {
|
||||
await notify("Homelab Manager — its own backup failed", `The scheduled backup of Homelab Manager's database didn't work: ${err instanceof Error ? err.message : String(err)}`).catch(() => {});
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The next time it will run, or null when it is off. */
|
||||
export function nextBackupRun(): Date | null {
|
||||
const next = currentJob?.nextInvocation();
|
||||
return next ? new Date(next.getTime()) : null;
|
||||
}
|
||||
|
||||
/** (Re)schedules the daily backup per the current settings. Call again after they change. */
|
||||
export async function scheduleSelfBackup(): Promise<void> {
|
||||
if (currentJob) {
|
||||
currentJob.cancel();
|
||||
currentJob = null;
|
||||
}
|
||||
const { selfBackup, notifications } = await getSettings();
|
||||
if (!selfBackup.enabled) {
|
||||
console.log("[selfBackup] daily backups are off, nothing scheduled");
|
||||
return;
|
||||
}
|
||||
currentJob = schedule.scheduleJob({ rule: cronForBackup(selfBackup.time), tz: notifications.timezone }, () => {
|
||||
void runScheduledBackup();
|
||||
});
|
||||
console.log(`Daily backup scheduled at ${selfBackup.time} (${notifications.timezone})`);
|
||||
}
|
||||
|
||||
/** Schedules it, and takes one soon if the app has been off through its time (or has never taken one). */
|
||||
export async function initSelfBackupScheduler(): Promise<void> {
|
||||
await scheduleSelfBackup();
|
||||
const { selfBackup } = await getSettings();
|
||||
if (!selfBackup.enabled) return;
|
||||
const newest = newestSnapshot();
|
||||
const overdue = !newest || Date.now() - Date.parse(newest.at) > 26 * 3600_000;
|
||||
// Only retry a failing one at the next scheduled time, not at every restart.
|
||||
const last = await lastAttempt();
|
||||
if (overdue && !(last && !last.ok && Date.now() - Date.parse(last.at) < 6 * 3600_000)) {
|
||||
setTimeout(() => void runScheduledBackup(), 30_000).unref();
|
||||
}
|
||||
}
|
||||
@@ -72,6 +72,17 @@ export function describeAudit(category: string, action: string, detail: Record<s
|
||||
return { title: "Admin link removed", tone: "neutral", kind: "change", detail: label };
|
||||
}
|
||||
}
|
||||
if (category === "alert") {
|
||||
const note = typeof detail.note === "string" && detail.note ? detail.note : null;
|
||||
const assignee = typeof detail.assignee === "string" ? detail.assignee : null;
|
||||
const lasts = typeof detail.hours === "number" ? `for ${detail.hours >= 48 ? `${Math.round(detail.hours / 24)} days` : `${detail.hours} h`}` : "until the alert is gone";
|
||||
switch (action) {
|
||||
case "ack":
|
||||
return { title: assignee ? `Acknowledged, handed to ${assignee}` : "Acknowledged", tone: "info", kind: "change", detail: [note, lasts].filter(Boolean).join(" · ") };
|
||||
case "unack":
|
||||
return { title: "Acknowledgement taken back", tone: "neutral", kind: "change", detail: null };
|
||||
}
|
||||
}
|
||||
if (category === "update_run") {
|
||||
const template = typeof detail.template === "string" ? detail.template : null;
|
||||
switch (action) {
|
||||
|
||||
@@ -129,6 +129,15 @@ export interface AgentSettings {
|
||||
autoUpdate: boolean;
|
||||
}
|
||||
|
||||
/** The daily backup of Homelab Manager's own database (see services/selfBackup.ts). On unless an administrator turns it off. */
|
||||
export interface SelfBackupSettings {
|
||||
enabled: boolean;
|
||||
/** "HH:MM", in the notification time zone. */
|
||||
time: string;
|
||||
/** How many of the daily backups are kept; the oldest beyond that are deleted. */
|
||||
keep: number;
|
||||
}
|
||||
|
||||
/** What the inventory needs to work out a running cost. Without a price per kWh it shows draw but no cost. */
|
||||
export interface InventorySettings {
|
||||
/** A short name shown after amounts ("SEK", "kr", "$"); may be blank. */
|
||||
@@ -190,6 +199,7 @@ export interface AppSettings {
|
||||
digest: DigestSettings;
|
||||
signIn: SignInSettings;
|
||||
inventory: InventorySettings;
|
||||
selfBackup: SelfBackupSettings;
|
||||
agents: AgentSettings;
|
||||
consistency: ConsistencySettings;
|
||||
nameGenerator: NameGeneratorSettings;
|
||||
@@ -227,6 +237,7 @@ const DEFAULTS: AppSettings = {
|
||||
digest: { enabled: false, day: 1, time: "08:00" },
|
||||
inventory: { currency: "", pricePerKwh: null },
|
||||
agents: { autoUpdate: false },
|
||||
selfBackup: { enabled: true, time: "03:30", keep: 14 },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
|
||||
@@ -20,6 +20,8 @@ import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
import { attentionCounts, loadAcks } from "./alertAcks.js";
|
||||
import { listIncidents } from "./incidents.js";
|
||||
|
||||
export const DIGEST_DAYS = 7;
|
||||
const DAY_MS = 24 * 3600_000;
|
||||
@@ -90,11 +92,30 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
if (report.counts.silenced > 0) held.push(`${report.counts.silenced} silenced by maintenance`);
|
||||
if (report.counts.grouped > 0) held.push(`${report.counts.grouped} folded under a server that is down`);
|
||||
if (held.length > 0) lines.push(`(${held.join(", ")})`);
|
||||
const acknowledged = attentionCounts(report.alerts, await loadAcks(now)).acknowledged;
|
||||
if (acknowledged > 0) lines.push(`(${acknowledged} of them acknowledged — somebody is on it)`);
|
||||
sections.push({ heading: "Right now", lines, attention: active.length > 0 || report.couldntCheck.length > 0 });
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the alerts:", err);
|
||||
}
|
||||
|
||||
// ── Incidents ──
|
||||
try {
|
||||
const since = now.getTime() - DIGEST_DAYS * DAY_MS;
|
||||
const all = await listIncidents({ now });
|
||||
const begun = all.filter((i) => Date.parse(i.openedAt) >= since);
|
||||
const ended = all.filter((i) => i.resolvedAt !== null && Date.parse(i.resolvedAt) >= since);
|
||||
const open = all.filter((i) => i.open);
|
||||
if (begun.length > 0 || ended.length > 0) {
|
||||
const lines = [`${plural(begun.length, "incident")} began, ${ended.length} ended, ${open.length} still open.`];
|
||||
const longest = [...ended].sort((a, b) => b.durationMs - a.durationMs).slice(0, 3);
|
||||
lines.push(...longest.map((i) => `- ${i.message} — lasted ${spanWords(i.durationMs / 60_000)}`));
|
||||
sections.push({ heading: "Incidents", lines, attention: open.length > 0 });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the incidents:", err);
|
||||
}
|
||||
|
||||
// ── Uptime ──
|
||||
try {
|
||||
const uptime = await getUptime(now, DIGEST_DAYS);
|
||||
|
||||
@@ -0,0 +1,453 @@
|
||||
/**
|
||||
* Acknowledging and handing over alerts, and the incident record: when an alert began and ended, and what happened around it.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-incidents-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, auditLog, maintenanceWindows, updateRuns, alertAcks, alertIncidents, incidentNotes } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const acks = await import("../src/services/alertAcks.js");
|
||||
const inc = await import("../src/services/incidents.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { getServerActivity } = await import("../src/services/serverActivity.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { buildDigest } = await import("../src/services/weeklyDigest.js");
|
||||
const { alertsRouter } = await import("../src/routes/alerts.js");
|
||||
const { incidentsRouter } = await import("../src/routes/incidents.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const T0 = new Date("2026-10-17T12:00:00.000Z");
|
||||
const at = (minutes: number) => new Date(T0.getTime() + minutes * 60_000);
|
||||
const alert = (id: string, extra: Record<string, unknown> = {}) =>
|
||||
({ id, severity: "warning" as const, category: "disk" as const, source: "Server", message: `${id} is a problem`, link: null, silenced: false, target: null, ...extra }) as never;
|
||||
|
||||
let adminId = 0;
|
||||
let operatorId = 0;
|
||||
let viewerId = 0;
|
||||
before(async () => {
|
||||
const rows = await db
|
||||
.insert(users)
|
||||
.values([
|
||||
{ oidcSub: "ada", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "olle", name: "Olle", role: "operator" },
|
||||
{ oidcSub: "vera", name: "Vera", role: "viewer" },
|
||||
])
|
||||
.returning();
|
||||
[adminId, operatorId, viewerId] = rows.map((r) => r.id);
|
||||
});
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
describe("acknowledging an alert", () => {
|
||||
test("a note, who has it and for how long are kept; a second one replaces the first; taking it back removes it", async () => {
|
||||
const first = await acks.setAck({ alertId: "disk:server:1:/", note: " looking at it ", assigneeId: operatorId, hours: 4, actor: "Ada" }, T0);
|
||||
assert.equal(first.before, null);
|
||||
assert.equal(first.ack.note, "looking at it");
|
||||
assert.deepEqual(first.ack.assignee, { id: operatorId, name: "Olle" });
|
||||
assert.equal(first.ack.ackedBy, "Ada");
|
||||
assert.equal(first.ack.until, at(240).toISOString());
|
||||
assert.equal(first.ack.active, true);
|
||||
|
||||
const second = await acks.setAck({ alertId: "disk:server:1:/", note: "", assigneeId: null, hours: null, actor: "Olle" }, at(10));
|
||||
assert.equal(second.before!.note, "looking at it");
|
||||
assert.equal(second.ack.assignee, null);
|
||||
assert.equal(second.ack.until, null, "until the alert is gone");
|
||||
assert.equal((await db.select().from(alertAcks)).length, 1, "one per alert");
|
||||
|
||||
assert.equal((await acks.clearAck("disk:server:1:/"))!.ackedBy, "Olle");
|
||||
assert.equal(await acks.clearAck("disk:server:1:/"), null);
|
||||
assert.equal((await acks.loadAcks()).size, 0);
|
||||
});
|
||||
|
||||
test("only people who can act can be handed an alert, and the note and the time are limited", async () => {
|
||||
const refused = async (input: Partial<Parameters<typeof acks.setAck>[0]>, code: string) =>
|
||||
assert.rejects(acks.setAck({ alertId: "x", note: "", assigneeId: null, hours: null, actor: "Ada", ...input }, T0), (e: unknown) => e instanceof acks.AckError && e.code === code, code);
|
||||
await refused({ assigneeId: viewerId }, "invalid_assignee");
|
||||
await refused({ assigneeId: 99999 }, "invalid_assignee");
|
||||
await refused({ note: "x".repeat(acks.MAX_NOTE + 1) }, "note_too_long");
|
||||
await refused({ hours: 0 }, "invalid_duration");
|
||||
await refused({ hours: acks.MAX_HOURS + 1 }, "invalid_duration");
|
||||
await refused({ hours: 1.5 }, "invalid_duration");
|
||||
assert.deepEqual((await acks.assignableUsers()).map((u) => u.name), ["Ada", "Olle"], "no viewers in the list");
|
||||
});
|
||||
|
||||
test("it runs out when it was meant to, and only then does the alert count as unattended again", async () => {
|
||||
await acks.setAck({ alertId: "a:1", note: "", assigneeId: null, hours: 2, actor: "Ada" }, T0);
|
||||
await acks.setAck({ alertId: "a:2", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
|
||||
const alerts = [
|
||||
{ id: "a:1", severity: "critical", silenced: false },
|
||||
{ id: "a:2", severity: "warning", silenced: false },
|
||||
{ id: "a:3", severity: "critical", silenced: false },
|
||||
{ id: "a:4", severity: "warning", silenced: true },
|
||||
] as { id: string; severity: "critical" | "warning" | "info"; silenced: boolean }[];
|
||||
assert.deepEqual(acks.attentionCounts(alerts, await acks.loadAcks(at(60))), { critical: 1, warning: 0, acknowledged: 2 }, "a:3 is the only one nobody has looked at; a silenced one is neither");
|
||||
const later = await acks.loadAcks(at(150));
|
||||
assert.equal(later.get("a:1")!.active, false);
|
||||
assert.deepEqual(acks.attentionCounts(alerts, later), { critical: 2, warning: 0, acknowledged: 1 }, "a:1 is unattended again; a:2 has no end");
|
||||
await db.delete(alertAcks);
|
||||
});
|
||||
|
||||
test("it goes when the alert has been gone for half an hour — not for a blip, and not if the alert is back", async () => {
|
||||
await acks.setAck({ alertId: "gone", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
|
||||
await acks.setAck({ alertId: "stays", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
|
||||
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(10)), 0, "ten minutes missing is a blip");
|
||||
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(29)), 0);
|
||||
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(31)), 1);
|
||||
assert.deepEqual([...(await acks.loadAcks()).keys()], ["stays"]);
|
||||
// a problem that returns later is a new one, nobody having looked at it
|
||||
assert.equal((await acks.loadAcks()).has("gone"), false);
|
||||
await db.delete(alertAcks);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the incident record", () => {
|
||||
const look = (alerts: unknown[], minutes: number, options: { blind?: number; inGrace?: boolean } = {}) => inc.recordAlertSnapshot({ alerts: alerts as never, blind: options.blind ?? 0 }, at(minutes), { inGrace: options.inGrace });
|
||||
const rows = () => db.select().from(alertIncidents).orderBy(alertIncidents.id);
|
||||
|
||||
test("an alert that appears begins an incident; the same look again changes nothing; information-only alerts aren't incidents", async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
|
||||
const r1 = await look([a, alert("info:thing", { severity: "info" })], 0);
|
||||
assert.equal(r1.opened.length, 1);
|
||||
assert.deepEqual((await look([a], 1)).opened, []);
|
||||
const [row] = await rows();
|
||||
assert.equal(row.alertKey, "disk:server:1:/");
|
||||
assert.equal(row.openedAt, at(0).toISOString());
|
||||
assert.equal(row.resolvedAt, null);
|
||||
assert.equal(row.targetName, "elsa");
|
||||
assert.equal(row.lastSeenAt, at(1).toISOString());
|
||||
assert.equal((await rows()).length, 1);
|
||||
});
|
||||
|
||||
test("it keeps the worst it got to and the latest wording", async () => {
|
||||
await look([alert("disk:server:1:/", { severity: "critical", message: "now 97% full", target: { type: "server", id: 1, name: "elsa" } })], 5);
|
||||
await look([alert("disk:server:1:/", { severity: "warning", message: "back to 91% full", target: { type: "server", id: 1, name: "elsa" } })], 6);
|
||||
const [row] = await rows();
|
||||
assert.equal(row.severity, "warning");
|
||||
assert.equal(row.peakSeverity, "critical");
|
||||
assert.equal(row.message, "back to 91% full");
|
||||
});
|
||||
|
||||
test("it is only called over after being missing for a couple of minutes in a row, and then it ended when it was first missing", async () => {
|
||||
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
|
||||
await look([], 10);
|
||||
let [row] = await rows();
|
||||
assert.equal(row.resolvedAt, null);
|
||||
assert.equal(row.missingSince, at(10).toISOString());
|
||||
await look([a], 11); // it was only a blip
|
||||
[row] = await rows();
|
||||
assert.equal(row.missingSince, null, "back before it was called over: the count starts again");
|
||||
assert.equal(row.resolvedAt, null);
|
||||
|
||||
await look([], 20);
|
||||
await look([], 21);
|
||||
[row] = await rows();
|
||||
assert.equal(row.resolvedAt, null, "one minute isn't enough");
|
||||
const done = await look([], 23);
|
||||
assert.equal(done.resolved.length, 1);
|
||||
[row] = await rows();
|
||||
assert.equal(row.resolvedAt, at(20).toISOString(), "it ended when it first wasn't there, not when that was confirmed");
|
||||
assert.equal(row.missingSince, null);
|
||||
});
|
||||
|
||||
test("when it comes back it is a new incident, and the old one stays as it was", async () => {
|
||||
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
|
||||
const r = await look([a], 60);
|
||||
assert.equal(r.opened.length, 1);
|
||||
const all = await rows();
|
||||
assert.equal(all.length, 2);
|
||||
assert.equal(all[0].resolvedAt, at(20).toISOString());
|
||||
assert.equal(all[1].resolvedAt, null);
|
||||
assert.equal(all[1].peakSeverity, "warning");
|
||||
});
|
||||
|
||||
test("after a check couldn't run it takes much longer to call anything over, and never in the first minutes after the app starts", async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const a = alert("proxmox-backup:1:pve1", { category: "backup" });
|
||||
await look([a], 0);
|
||||
await look([], 5, { blind: 1 });
|
||||
await look([], 20, { blind: 1 });
|
||||
assert.equal((await rows())[0].resolvedAt, null, "fifteen minutes missing isn't enough while a check is failing");
|
||||
assert.equal((await look([], 36, { blind: 1 })).resolved.length, 1);
|
||||
|
||||
await db.delete(alertIncidents);
|
||||
await look([a], 100);
|
||||
await look([], 101, { inGrace: true });
|
||||
await look([], 200, { inGrace: true });
|
||||
const [row] = await rows();
|
||||
assert.equal(row.resolvedAt, null, "server alerts are held back on purpose for a while after the app starts, so missing then means nothing");
|
||||
assert.equal(row.missingSince, null);
|
||||
});
|
||||
|
||||
test("it is recorded from the alerts as they are collected, without anyone asking, and the Alerts page links to it", async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const [s] = await db.insert(servers).values({ name: "quiet", apiTokenHash: hashToken("q"), apiTokenPrefix: "q", lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString() }).returning();
|
||||
invalidateAlertsCache();
|
||||
await getAlerts(true);
|
||||
let found: typeof alertIncidents.$inferSelect | undefined;
|
||||
for (let i = 0; i < 40 && !found; i++) {
|
||||
found = (await rows()).find((r) => r.alertKey === `offline:offline:server:${s.id}`);
|
||||
if (!found) await new Promise((r) => setTimeout(r, 50));
|
||||
}
|
||||
assert.ok(found, "an incident for the offline server");
|
||||
assert.equal(found.severity, "critical");
|
||||
assert.deepEqual([found.targetType, found.targetId, found.targetName], ["server", s.id, "quiet"]);
|
||||
assert.equal((await inc.openIncidentIds([found.alertKey])).get(found.alertKey), found.id);
|
||||
await db.delete(servers).where(eq(servers.id, s.id));
|
||||
});
|
||||
|
||||
test("resolved ones are kept for a long while and then removed with their notes; open ones never", async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const old = new Date(Date.now() - (inc.KEEP_DAYS + 5) * 86_400_000).toISOString();
|
||||
const [oldRow] = await db.insert(alertIncidents).values({ alertKey: "old", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "old", openedAt: old, resolvedAt: old, lastSeenAt: old }).returning();
|
||||
await db.insert(incidentNotes).values({ incidentId: oldRow.id, author: "Ada", body: "n", createdAt: old });
|
||||
const [openRow] = await db.insert(alertIncidents).values({ alertKey: "ancient-open", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "still open", openedAt: old, lastSeenAt: old }).returning();
|
||||
assert.equal(await inc.pruneIncidents(), 1);
|
||||
assert.deepEqual((await rows()).map((r) => r.id), [openRow.id]);
|
||||
assert.equal((await db.select().from(incidentNotes)).length, 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the timeline around an incident", () => {
|
||||
let serverId = 0;
|
||||
let incidentId = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("e"), apiTokenPrefix: "e" }).returning();
|
||||
serverId = s.id;
|
||||
const [row] = await db
|
||||
.insert(alertIncidents)
|
||||
.values({ alertKey: `offline:offline:server:${serverId}`, severity: "critical", peakSeverity: "critical", category: "offline", source: "Server", message: "elsa hasn't reported for 2 h", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(0).toISOString(), resolvedAt: at(95).toISOString(), lastSeenAt: at(95).toISOString() })
|
||||
.returning();
|
||||
incidentId = row.id;
|
||||
const stamp = (m: number) => at(m).toISOString().slice(0, 19).replace("T", " ");
|
||||
const audit = (m: number, category: string, action: string, detail: object, actor = "system", target: object = { targetType: "server", targetId: String(serverId) }) =>
|
||||
db.insert(auditLog).values({ actorLabel: actor, category, action, detail: JSON.stringify(detail), createdAt: stamp(m), ...target });
|
||||
await audit(1, "server", "went_offline", { message: "elsa hasn't reported for 2 h" });
|
||||
await audit(5, "alert", "ack", { alertId: row.alertKey, message: "x", note: "power cut in the cellar", assignee: "Olle", hours: 4 }, "Ada");
|
||||
await audit(6, "alert", "ack", { alertId: "some:other:alert", note: "not this one" }, "Ada");
|
||||
await audit(30, "server", "update", { name: "elsa" }, "Ada");
|
||||
await audit(90, "server", "back_online", {});
|
||||
await audit(40, "server", "update", {}, "Ada", { targetType: "server", targetId: "9999" }); // another server
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "UPS swap", startedAt: at(20).toISOString(), endsAt: at(60).toISOString(), createdBy: "Ada" });
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "withdrawn", startedAt: at(25).toISOString(), endsAt: at(25).toISOString() });
|
||||
await db.insert(updateRuns).values({ serverId, integrationId: 1, projectId: 1, templateId: 1, templateName: "Apply updates", hostLimit: "elsa", status: "done", message: "No updates are waiting now; no restart is needed.", startedBy: "Ada", startedAt: at(61).toISOString(), finishedAt: at(80).toISOString() });
|
||||
await db.insert(alertIncidents).values({ alertKey: `disk:server:${serverId}:/`, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "/ is 93% full", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(8).toISOString(), lastSeenAt: at(8).toISOString() });
|
||||
await db.insert(alertIncidents).values({ alertKey: `disk:server:${serverId}:/old`, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "long ago", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(-600).toISOString(), lastSeenAt: at(-600).toISOString() });
|
||||
await inc.addNote(incidentId, "Olle", "Confirmed: the building lost power.", at(15));
|
||||
});
|
||||
|
||||
test("it puts everything in the order it happened, and only what belongs to this incident", async () => {
|
||||
const d = (await inc.getIncident(incidentId, at(100)))!;
|
||||
assert.equal(d.incident.open, false);
|
||||
assert.equal(d.incident.durationMs, 95 * 60_000);
|
||||
const titles = d.timeline.map((e) => `${e.at.slice(11, 16)} ${e.kind}: ${e.title}`);
|
||||
assert.deepEqual(titles, [
|
||||
"12:00 alert: Alert raised (critical)",
|
||||
"12:01 health: Stopped reporting — marked offline",
|
||||
"12:05 ack: Acknowledged, handed to Olle",
|
||||
"12:08 related: Another alert was raised: disk (warning)",
|
||||
"12:15 note: Olle wrote a note",
|
||||
"12:20 maintenance: Maintenance began — alerts silenced",
|
||||
"12:30 change: Details changed",
|
||||
"12:55 maintenance: Maintenance ended".replace("12:55", "13:00"),
|
||||
"13:01 run: Update run started: Apply updates",
|
||||
"13:20 run: Update run finished",
|
||||
"13:30 health: Reporting again",
|
||||
"13:35 alert: Resolved",
|
||||
]);
|
||||
});
|
||||
|
||||
test("the details say what was written and by whom", async () => {
|
||||
const d = (await inc.getIncident(incidentId, at(100)))!;
|
||||
const ack = d.timeline.find((e) => e.kind === "ack")!;
|
||||
assert.equal(ack.actor, "Ada");
|
||||
assert.match(ack.detail!, /power cut in the cellar · for 4 h/);
|
||||
const note = d.timeline.find((e) => e.kind === "note")!;
|
||||
assert.equal(note.detail, "Confirmed: the building lost power.");
|
||||
assert.ok(note.noteId);
|
||||
const related = d.timeline.find((e) => e.kind === "related")!;
|
||||
assert.match(related.detail!, /93% full/);
|
||||
assert.ok(related.incidentId);
|
||||
assert.equal(d.timeline.find((e) => e.title === "Resolved")!.detail, "It lasted 1.6 h.");
|
||||
assert.equal(d.ack, null, "no acknowledgement to show on one that is over");
|
||||
assert.equal(d.notes.length, 1);
|
||||
assert.equal((await inc.getIncident(999999)), null);
|
||||
});
|
||||
|
||||
test("an open one runs to now and shows the acknowledgement in force", async () => {
|
||||
const [row] = await db.select().from(alertIncidents).where(eq(alertIncidents.alertKey, `disk:server:${serverId}:/`));
|
||||
await acks.setAck({ alertId: row.alertKey, note: "freeing space", assigneeId: operatorId, hours: null, actor: "Ada" }, at(9));
|
||||
const d = (await inc.getIncident(row.id, at(120)))!;
|
||||
assert.equal(d.incident.open, true);
|
||||
assert.equal(d.incident.durationMs, 112 * 60_000);
|
||||
assert.equal(d.ack!.note, "freeing space");
|
||||
assert.equal(d.ack!.assignee!.name, "Olle");
|
||||
assert.ok(!d.timeline.some((e) => e.title === "Resolved"));
|
||||
await db.delete(alertAcks);
|
||||
});
|
||||
|
||||
test("the list has the open ones first, then the rest newest first, with how many notes each has", async () => {
|
||||
const list = await inc.listIncidents({ now: at(120) });
|
||||
assert.equal(list[0].open, true);
|
||||
const firstClosed = list.findIndex((i) => !i.open);
|
||||
assert.ok(list.slice(0, firstClosed).every((i) => i.open) && list.slice(firstClosed).every((i) => !i.open));
|
||||
assert.equal(list.find((i) => i.id === incidentId)!.notes, 1);
|
||||
assert.deepEqual(list.filter((i) => !i.open).map((i) => i.openedAt), list.filter((i) => !i.open).map((i) => i.openedAt).sort().reverse());
|
||||
});
|
||||
});
|
||||
|
||||
describe("in the weekly digest", () => {
|
||||
test("the week's incidents are counted, the longest are named, and acknowledged alerts are said to be", async () => {
|
||||
const digest = await buildDigest(at(120));
|
||||
assert.match(digest.message, /INCIDENTS/);
|
||||
assert.match(digest.message, /\d+ incidents? began, 1 ended, \d+ still open/);
|
||||
assert.match(digest.message, /elsa hasn't reported for 2 h — lasted 1 h 35 min/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "ada";
|
||||
let serverId = 0;
|
||||
let alertId = "";
|
||||
let incidentId = 0;
|
||||
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
before(async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const [s] = await db.insert(servers).values({ name: "zoe", apiTokenHash: hashToken("z"), apiTokenPrefix: "z", lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString() }).returning();
|
||||
serverId = s.id;
|
||||
alertId = `offline:offline:server:${serverId}`.replace("offline:offline", "offline:offline");
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/alerts", alertsRouter);
|
||||
a.use("/api/incidents", incidentsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
invalidateAlertsCache();
|
||||
const report = await (await call("GET", "/alerts?refresh=1")).json();
|
||||
alertId = report.alerts.find((x: { id: string }) => x.id.includes(`:server:${serverId}`)).id;
|
||||
for (let i = 0; i < 40 && !incidentId; i++) {
|
||||
incidentId = (await db.select().from(alertIncidents)).find((r) => r.alertKey === alertId)?.id ?? 0;
|
||||
if (!incidentId) await new Promise((r) => setTimeout(r, 50));
|
||||
}
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
test("operators acknowledge and hand over; viewers can look but not; an alert that isn't there can't be", async () => {
|
||||
who = "vera";
|
||||
assert.equal((await call("PUT", "/alerts/ack", { alertId })).status, 403);
|
||||
assert.equal((await call("GET", "/alerts/assignees")).status, 403);
|
||||
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 403);
|
||||
assert.equal((await call("GET", "/alerts")).status, 200);
|
||||
|
||||
who = "olle";
|
||||
const names = (await (await call("GET", "/alerts/assignees")).json()).users.map((u: { name: string }) => u.name);
|
||||
assert.deepEqual(names, ["Ada", "Olle"]);
|
||||
assert.equal((await call("PUT", "/alerts/ack", { alertId: "no:such:alert" })).status, 404);
|
||||
assert.equal((await call("PUT", "/alerts/ack", { alertId: "" })).status, 400);
|
||||
assert.equal((await call("PUT", "/alerts/ack", { alertId, hours: 0 })).status, 400);
|
||||
const bad = await call("PUT", "/alerts/ack", { alertId, assigneeId: viewerId });
|
||||
assert.equal(bad.status, 400);
|
||||
assert.equal((await bad.json()).error, "invalid_assignee");
|
||||
|
||||
const res = await call("PUT", "/alerts/ack", { alertId, note: "power cut", assigneeId: adminId, hours: 6 });
|
||||
assert.equal(res.status, 200);
|
||||
const { ack } = await res.json();
|
||||
assert.equal(ack.assignee.name, "Ada");
|
||||
assert.equal(ack.ackedBy, "Olle");
|
||||
assert.equal(ack.active, true);
|
||||
});
|
||||
|
||||
test("the Alerts page shows it on the alert, leaves it out of what needs attention, and links to the incident", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await (await call("GET", "/alerts")).json();
|
||||
const a = report.alerts.find((x: { id: string }) => x.id === alertId);
|
||||
assert.equal(a.ack.note, "power cut");
|
||||
assert.equal(a.incidentId, incidentId);
|
||||
assert.ok(report.counts.acknowledged >= 1);
|
||||
assert.ok(report.counts.attention.critical < report.counts.critical, "the whole count is still the whole count; attention leaves out the acknowledged");
|
||||
const metrics = await buildMetrics();
|
||||
assert.match(metrics, /homelab_alerts_acknowledged [1-9]/);
|
||||
assert.match(metrics, /homelab_incidents_open [1-9]/);
|
||||
});
|
||||
|
||||
test("it is in the audit log and in the server's history, and taking it back is too", async () => {
|
||||
const entry = (await db.select().from(auditLog)).filter((e) => e.category === "alert" && e.action === "ack").pop()!;
|
||||
assert.equal(entry.targetType, "server");
|
||||
assert.equal(entry.targetId, String(serverId));
|
||||
assert.equal(JSON.parse(entry.detail!).assignee, "Ada");
|
||||
const activity = await getServerActivity(serverId);
|
||||
assert.ok(activity.some((e) => e.title === "Acknowledged, handed to Ada" && /power cut/.test(e.detail ?? "")));
|
||||
|
||||
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 204);
|
||||
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 404);
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await (await call("GET", "/alerts")).json()).alerts.find((x: { id: string }) => x.id === alertId).ack, null);
|
||||
assert.ok((await getServerActivity(serverId)).some((e) => e.title === "Acknowledgement taken back"));
|
||||
});
|
||||
|
||||
test("everyone signed in can read the incidents; operators write notes; a note can be removed by its author or an administrator", async () => {
|
||||
who = "vera";
|
||||
const list = await (await call("GET", "/incidents")).json();
|
||||
assert.ok(list.incidents.some((i: { id: number }) => i.id === incidentId));
|
||||
assert.equal((await call("GET", `/incidents/${incidentId}`)).status, 200);
|
||||
assert.equal((await call("GET", "/incidents/999999")).status, 404);
|
||||
assert.equal((await call("GET", "/incidents/x")).status, 400);
|
||||
assert.equal((await call("POST", `/incidents/${incidentId}/notes`, { body: "hello" })).status, 403);
|
||||
|
||||
who = "olle";
|
||||
assert.equal((await call("POST", `/incidents/${incidentId}/notes`, { body: " " })).status, 400);
|
||||
assert.equal((await call("POST", "/incidents/999999/notes", { body: "x" })).status, 404);
|
||||
const made = await call("POST", `/incidents/${incidentId}/notes`, { body: "It was the UPS." });
|
||||
assert.equal(made.status, 201);
|
||||
const noteId = (await made.json()).id;
|
||||
const detail = await (await call("GET", `/incidents/${incidentId}`)).json();
|
||||
assert.ok(detail.timeline.some((e: { kind: string; detail: string }) => e.kind === "note" && e.detail === "It was the UPS."));
|
||||
|
||||
// another operator can't remove it, its author and an administrator can
|
||||
await db.insert(users).values({ oidcSub: "olga", name: "Olga", role: "operator" });
|
||||
who = "olga";
|
||||
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 403);
|
||||
who = "olle";
|
||||
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 204);
|
||||
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 404);
|
||||
const again = await (await call("POST", `/incidents/${incidentId}/notes`, { body: "second" })).json();
|
||||
who = "ada";
|
||||
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${again.id}`)).status, 204);
|
||||
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "incident").map((e) => e.action);
|
||||
assert.ok(actions.includes("note") && actions.includes("note_removed"));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,453 @@
|
||||
/**
|
||||
* Backups of Homelab Manager itself: taking them, checking them, keeping the newest, staging and applying a restore, the routes around
|
||||
* them, and what the alerts and /metrics say about how current they are.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-selfbackup-"));
|
||||
process.env.DATABASE_PATH = join(dir, "data", "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { createClient } = await import("@libsql/client");
|
||||
const { db, client } = await import("../src/db/client.js");
|
||||
const { users, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const sb = await import("../src/services/selfBackup.js");
|
||||
const { runScheduledBackup, cronForBackup } = await import("../src/services/selfBackupScheduler.js");
|
||||
const { applyPendingRestore } = await import("../src/db/applyRestore.js");
|
||||
const { PENDING_FILE, PENDING_NOTE, RESULT_NOTE } = await import("../src/db/restoreFiles.js");
|
||||
const { dbBackupsRouter } = await import("../src/routes/dbBackups.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { getSettings, updateSettings, setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const schedule = (await import("node-schedule")).default;
|
||||
|
||||
await runMigrations();
|
||||
after(async () => {
|
||||
await schedule.gracefulShutdown();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const at = (iso: string) => new Date(iso);
|
||||
/** Opens a copy of a backup, never the backup itself: an opened database can't be deleted on some systems until the process ends. */
|
||||
const open = (file: string) => {
|
||||
const copy = join(mkdtempSync(join(dir, "copy-")), "db.sqlite");
|
||||
copyFileSync(file, copy);
|
||||
return createClient({ url: `file:${copy}` });
|
||||
};
|
||||
const wipeBackups = () => rmSync(sb.backupDir(), { recursive: true, force: true });
|
||||
|
||||
describe("taking and checking a backup", () => {
|
||||
test("a backup is a consistent copy that opens, passes SQLite's check, and holds what was there", async () => {
|
||||
await db.insert(users).values({ oidcSub: "ada", name: "Ada", role: "admin" });
|
||||
const { info, check } = await sb.createSnapshot("manual", at("2026-10-17T12:00:00Z"));
|
||||
assert.equal(info.name, "homelab-manager-2026-10-17-12-00-00-manual.sqlite");
|
||||
assert.equal(info.kind, "manual");
|
||||
assert.ok(info.sizeBytes > 10_000);
|
||||
assert.equal(check.ok, true);
|
||||
assert.equal(check.integrity, "ok");
|
||||
assert.equal(check.migrations, sb.appMigrationCount());
|
||||
assert.equal(check.counts!.users, 1);
|
||||
|
||||
// and it really holds it: open the file on its own and read the row back
|
||||
const copy = open(join(sb.backupDir(), info.name));
|
||||
const rows = (await copy.execute("select name from users")).rows;
|
||||
copy.close();
|
||||
assert.deepEqual(rows.map((r) => r.name), ["Ada"]);
|
||||
});
|
||||
|
||||
test("changes made after it was taken are not in it", async () => {
|
||||
const { info } = await sb.createSnapshot("manual", at("2026-10-17T12:01:00Z"));
|
||||
await db.insert(users).values({ oidcSub: "bo", name: "Bo", role: "viewer" });
|
||||
const copy = open(join(sb.backupDir(), info.name));
|
||||
assert.equal(Number((await copy.execute("select count(*) as n from users")).rows[0].n), 1);
|
||||
copy.close();
|
||||
await db.delete(users).where((await import("drizzle-orm")).eq(users.oidcSub, "bo"));
|
||||
});
|
||||
|
||||
test("two taken in the same second don't collide", async () => {
|
||||
const a = await sb.createSnapshot("manual", at("2026-10-17T13:00:00Z"));
|
||||
const b = await sb.createSnapshot("manual", at("2026-10-17T13:00:00Z"));
|
||||
assert.notEqual(a.info.name, b.info.name);
|
||||
assert.match(b.info.name, /-manual-2\.sqlite$/);
|
||||
});
|
||||
|
||||
test("files that aren't a usable backup are said to be, and why", async () => {
|
||||
const junk = join(dir, "junk.sqlite");
|
||||
writeFileSync(junk, "this is not a database at all");
|
||||
assert.match((await sb.checkSnapshotFile(junk)).problem!, /isn't a database file/);
|
||||
assert.match((await sb.checkSnapshotFile(join(dir, "missing.sqlite"))).problem!, /isn't there/);
|
||||
|
||||
// a real SQLite file that isn't ours
|
||||
const other = join(dir, "other.sqlite");
|
||||
const c = createClient({ url: `file:${other}` });
|
||||
await c.execute("create table cats (name text)");
|
||||
c.close();
|
||||
assert.match((await sb.checkSnapshotFile(other)).problem!, /isn't a Homelab Manager database/);
|
||||
|
||||
// one of ours, made by a newer version (more schema changes than this one has)
|
||||
const newer = join(dir, "newer.sqlite");
|
||||
await client.execute(`VACUUM INTO '${newer.replace(/'/g, "''")}'`);
|
||||
const n = createClient({ url: `file:${newer}` });
|
||||
for (let i = 0; i < 3; i++) await n.execute({ sql: "insert into __drizzle_migrations (hash, created_at) values (?, ?)", args: [`future-${i}`, 9_999_999_999_999 + i] });
|
||||
n.close();
|
||||
const check = await sb.checkSnapshotFile(newer);
|
||||
assert.equal(check.ok, false);
|
||||
assert.equal(check.newerThanApp, true);
|
||||
assert.match(check.problem!, /newer version/);
|
||||
assert.ok(check.migrations! > check.appMigrations);
|
||||
});
|
||||
|
||||
test("an older one is fine: this version brings it up to date when it starts", async () => {
|
||||
const older = join(dir, "older.sqlite");
|
||||
await client.execute(`VACUUM INTO '${older.replace(/'/g, "''")}'`);
|
||||
const o = createClient({ url: `file:${older}` });
|
||||
await o.execute("delete from __drizzle_migrations where rowid = (select max(rowid) from __drizzle_migrations)");
|
||||
o.close();
|
||||
const check = await sb.checkSnapshotFile(older);
|
||||
assert.equal(check.ok, true);
|
||||
assert.equal(check.migrations, sb.appMigrationCount() - 1);
|
||||
});
|
||||
|
||||
test("only names of our own files are ever acted on, whatever is asked for", () => {
|
||||
for (const bad of ["../test.sqlite", "..\\test.sqlite", "homelab-manager-2026-10-17-12-00-00.sqlite/../../x", "homelab-manager-x.sqlite", "test.sqlite", "homelab-manager-2026-10-17-12-00-00.sqlite.bak", "/etc/passwd", ""]) {
|
||||
assert.equal(sb.isSnapshotName(bad), false, bad);
|
||||
assert.equal(sb.snapshotPath(bad), null, bad);
|
||||
assert.equal(sb.deleteSnapshot(bad), false, bad);
|
||||
}
|
||||
for (const good of ["homelab-manager-2026-10-17-12-00-00.sqlite", "homelab-manager-2026-10-17-12-00-00-manual.sqlite", "homelab-manager-2026-10-17-12-00-00-pre-restore-2.sqlite", "homelab-manager-2026-10-17-12-00-00-uploaded.sqlite"]) {
|
||||
assert.equal(sb.isSnapshotName(good), true, good);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("keeping the newest", () => {
|
||||
const fake = (name: string, mtime: string) => {
|
||||
mkdirSync(sb.backupDir(), { recursive: true });
|
||||
const f = join(sb.backupDir(), name);
|
||||
writeFileSync(f, "x");
|
||||
utimesSync(f, at(mtime), at(mtime));
|
||||
};
|
||||
const names = () => sb.listSnapshots().map((s) => s.name);
|
||||
|
||||
test("the newest N of the daily ones stay, and each other kind has its own, smaller allowance", () => {
|
||||
wipeBackups();
|
||||
for (let d = 1; d <= 6; d++) fake(`homelab-manager-2026-10-0${d}-03-30-00.sqlite`, `2026-10-0${d}T03:30:00Z`);
|
||||
for (let d = 1; d <= 12; d++) fake(`homelab-manager-2026-10-${String(d).padStart(2, "0")}-09-00-00-manual.sqlite`, `2026-10-${String(d).padStart(2, "0")}T09:00:00Z`);
|
||||
for (let d = 1; d <= 5; d++) fake(`homelab-manager-2026-10-0${d}-10-00-00-pre-restore.sqlite`, `2026-10-0${d}T10:00:00Z`);
|
||||
const removed = sb.pruneSnapshots(3);
|
||||
const left = names();
|
||||
assert.equal(left.filter((n) => !/-(manual|pre-restore|uploaded)/.test(n)).length, 3);
|
||||
assert.equal(left.filter((n) => n.includes("-manual")).length, 10);
|
||||
assert.equal(left.filter((n) => n.includes("-pre-restore")).length, 3);
|
||||
assert.equal(removed.length, 3 + 2 + 2);
|
||||
assert.ok(left.includes("homelab-manager-2026-10-06-03-30-00.sqlite") && !left.includes("homelab-manager-2026-10-01-03-30-00.sqlite"), "the oldest went, not the newest");
|
||||
// other files in the folder are left alone
|
||||
writeFileSync(join(sb.backupDir(), "notes.txt"), "mine");
|
||||
sb.pruneSnapshots(1);
|
||||
assert.ok(existsSync(join(sb.backupDir(), "notes.txt")));
|
||||
wipeBackups();
|
||||
});
|
||||
|
||||
test("a scheduled run takes one and trims to what is kept, and records that it worked", async () => {
|
||||
wipeBackups();
|
||||
await updateSettings({ selfBackup: { keep: 2 } });
|
||||
for (const d of ["2026-10-14T03:30:00Z", "2026-10-15T03:30:00Z", "2026-10-16T03:30:00Z"]) assert.ok(await runScheduledBackup(at(d)));
|
||||
assert.deepEqual(names(), ["homelab-manager-2026-10-16-03-30-00.sqlite", "homelab-manager-2026-10-15-03-30-00.sqlite"]);
|
||||
const last = await sb.lastAttempt();
|
||||
assert.equal(last!.ok, true);
|
||||
assert.equal(last!.kind, "scheduled");
|
||||
assert.equal(cronForBackup("03:30"), "30 3 * * *");
|
||||
assert.equal(cronForBackup("23:05"), "5 23 * * *");
|
||||
await updateSettings({ selfBackup: { keep: 14 } });
|
||||
});
|
||||
|
||||
test("a backup that can't be made is recorded as having failed, says why, and leaves no half file", async () => {
|
||||
wipeBackups();
|
||||
const realExecute = client.execute.bind(client);
|
||||
(client as unknown as { execute: unknown }).execute = async (stmt: unknown) => {
|
||||
if (typeof stmt === "string" && stmt.startsWith("VACUUM INTO")) throw new Error("SQLITE_FULL: database or disk is full");
|
||||
return realExecute(stmt as never);
|
||||
};
|
||||
try {
|
||||
await assert.rejects(sb.createSnapshot("scheduled", at("2026-10-17T03:30:00Z")), /SQLITE_FULL/);
|
||||
const last = await sb.lastAttempt();
|
||||
assert.equal(last!.ok, false);
|
||||
assert.match(last!.error!, /disk is full/);
|
||||
assert.deepEqual(sb.listSnapshots(), [], "nothing half-written is left in the backups folder");
|
||||
assert.equal(await runScheduledBackup(at("2026-10-17T03:31:00Z")), null, "the scheduled run swallows it (and says so through the channels)");
|
||||
} finally {
|
||||
(client as unknown as { execute: unknown }).execute = realExecute;
|
||||
}
|
||||
assert.ok(await runScheduledBackup(at("2026-10-17T03:32:00Z")), "and the next one works");
|
||||
assert.equal((await sb.lastAttempt())!.ok, true);
|
||||
wipeBackups();
|
||||
});
|
||||
});
|
||||
|
||||
describe("restoring", () => {
|
||||
test("a staged restore is checked first, kept beside the database, can be cancelled, and refuses what isn't usable", async () => {
|
||||
wipeBackups();
|
||||
const { info } = await sb.createSnapshot("manual", at("2026-10-17T15:00:00Z"));
|
||||
assert.equal(sb.stagedRestore(), null);
|
||||
|
||||
await assert.rejects(sb.stageRestore("../../etc/passwd", "Ada"), /no backup with that name/);
|
||||
await assert.rejects(sb.stageRestore("homelab-manager-2026-01-01-00-00-00.sqlite", "Ada"), /no backup with that name/);
|
||||
writeFileSync(join(sb.backupDir(), "homelab-manager-2026-10-17-16-00-00-uploaded.sqlite"), "garbage");
|
||||
await assert.rejects(sb.stageRestore("homelab-manager-2026-10-17-16-00-00-uploaded.sqlite", "Ada"), /isn't a database file/);
|
||||
assert.equal(sb.stagedRestore(), null, "a refused one stages nothing");
|
||||
|
||||
const check = await sb.stageRestore(info.name, "Ada", at("2026-10-17T15:05:00Z"));
|
||||
assert.equal(check.ok, true);
|
||||
const staged = sb.stagedRestore()!;
|
||||
assert.deepEqual(staged, { from: info.name, stagedAt: "2026-10-17T15:05:00.000Z", by: "Ada" });
|
||||
const dataDir = join(dir, "data");
|
||||
assert.ok(existsSync(join(dataDir, PENDING_FILE)));
|
||||
assert.equal(readdirSync(dataDir).some((f) => f.endsWith(".copying")), false, "no half-copied file is left");
|
||||
assert.equal(statSync(join(dataDir, PENDING_FILE)).size, info.sizeBytes);
|
||||
|
||||
assert.equal(sb.cancelStagedRestore(), true);
|
||||
assert.equal(sb.stagedRestore(), null);
|
||||
assert.equal(existsSync(join(dataDir, PENDING_FILE)), false);
|
||||
assert.equal(sb.cancelStagedRestore(), false);
|
||||
wipeBackups();
|
||||
});
|
||||
|
||||
test("at start-up the staged file replaces the database, the old one is kept, and nothing happens twice", () => {
|
||||
const folder = mkdtempSync(join(dir, "apply-"));
|
||||
const dbFile = join(folder, "app.sqlite");
|
||||
writeFileSync(dbFile, "OLD DATABASE");
|
||||
writeFileSync(`${dbFile}-wal`, "stale journal");
|
||||
writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE");
|
||||
writeFileSync(join(folder, PENDING_NOTE), JSON.stringify({ from: "homelab-manager-2026-10-01-03-30-00.sqlite", by: "Ada" }));
|
||||
|
||||
const result = applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z"))!;
|
||||
assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE");
|
||||
assert.equal(existsSync(`${dbFile}-wal`), false, "a journal from the old file would be taken for the new one's");
|
||||
assert.equal(result.keptAs, "homelab-manager-2026-10-17-16-00-00-pre-restore.sqlite");
|
||||
assert.equal(readFileSync(join(folder, "backups", result.keptAs!), "utf8"), "OLD DATABASE", "what was in use is kept, not deleted");
|
||||
assert.equal(result.from, "homelab-manager-2026-10-01-03-30-00.sqlite");
|
||||
assert.equal(result.by, "Ada");
|
||||
assert.equal(existsSync(join(folder, PENDING_FILE)), false);
|
||||
assert.equal(existsSync(join(folder, PENDING_NOTE)), false);
|
||||
assert.deepEqual(JSON.parse(readFileSync(join(folder, RESULT_NOTE), "utf8")), result);
|
||||
|
||||
assert.equal(applyPendingRestore(dbFile, at("2026-10-17T16:01:00Z")), null, "nothing staged any more");
|
||||
assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE");
|
||||
});
|
||||
|
||||
test("with no database there yet (a fresh machine), the staged one simply becomes it", () => {
|
||||
const folder = mkdtempSync(join(dir, "fresh-"));
|
||||
const dbFile = join(folder, "app.sqlite");
|
||||
writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE");
|
||||
const result = applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z"))!;
|
||||
assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE");
|
||||
assert.equal(result.keptAs, null);
|
||||
assert.equal(result.from, "an uploaded backup", "no note, no name — said plainly");
|
||||
});
|
||||
|
||||
test("a restore that can't be completed leaves the database exactly as it was", () => {
|
||||
// the folder for keeping the old one can't be made (a file is in its place)
|
||||
const folder = mkdtempSync(join(dir, "fail-"));
|
||||
const dbFile = join(folder, "app.sqlite");
|
||||
writeFileSync(dbFile, "OLD DATABASE");
|
||||
writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE");
|
||||
writeFileSync(join(folder, "backups"), "I am a file, not a folder");
|
||||
assert.equal(applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z")), null);
|
||||
assert.equal(readFileSync(dbFile, "utf8"), "OLD DATABASE", "the database is as it was");
|
||||
assert.equal(readFileSync(join(folder, PENDING_FILE), "utf8"), "NEW DATABASE", "and the staged one is still there to try again");
|
||||
assert.equal(existsSync(join(folder, RESULT_NOTE)), false);
|
||||
|
||||
// something staged that isn't a file is never put in the database's place
|
||||
const odd = mkdtempSync(join(dir, "odd-"));
|
||||
writeFileSync(join(odd, "app.sqlite"), "OLD DATABASE");
|
||||
mkdirSync(join(odd, PENDING_FILE));
|
||||
assert.equal(applyPendingRestore(join(odd, "app.sqlite"), at("2026-10-17T16:00:00Z")), null);
|
||||
assert.equal(readFileSync(join(odd, "app.sqlite"), "utf8"), "OLD DATABASE");
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "ada";
|
||||
const call = (method: string, path: string, body?: unknown, raw?: Buffer) =>
|
||||
fetch(`${base}${path}`, raw ? { method, headers: { "content-type": "application/octet-stream" }, body: new Uint8Array(raw) } : { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values({ oidcSub: "operator", name: "Olle", role: "operator" });
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/db-backups", dbBackupsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/db-backups`;
|
||||
wipeBackups();
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
test("everything about backups is for administrators only", async () => {
|
||||
who = "operator";
|
||||
for (const [method, path] of [["GET", "/"], ["POST", "/"], ["PUT", "/settings"], ["POST", "/upload"], ["GET", "/x/download"], ["POST", "/x/restore"], ["POST", "/x/check"], ["DELETE", "/x"], ["POST", "/restore/cancel"]] as const) {
|
||||
assert.equal((await call(method, path, method === "GET" || method === "DELETE" ? undefined : {})).status, 403, `${method} ${path}`);
|
||||
}
|
||||
who = "ada";
|
||||
assert.equal((await call("GET", "/")).status, 200);
|
||||
});
|
||||
|
||||
test("settings are checked and kept, and change what is kept at once", async () => {
|
||||
assert.equal((await call("PUT", "/settings", { enabled: true, time: "25:00", keep: 7 })).status, 400);
|
||||
assert.equal((await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 0 })).status, 400);
|
||||
assert.equal((await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 500 })).status, 400);
|
||||
const res = await call("PUT", "/settings", { enabled: true, time: "04:15", keep: 7 });
|
||||
assert.equal(res.status, 200);
|
||||
const body = await res.json();
|
||||
assert.deepEqual(body.settings, { enabled: true, time: "04:15", keep: 7 });
|
||||
assert.ok(body.nextRun, "it says when the next one is");
|
||||
assert.equal((await getSettings()).selfBackup.time, "04:15");
|
||||
const off = await (await call("PUT", "/settings", { enabled: false, time: "04:15", keep: 7 })).json();
|
||||
assert.equal(off.nextRun, null, "off means nothing is scheduled");
|
||||
await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 14 });
|
||||
const entry = (await db.select().from(auditLog)).filter((e) => e.category === "db_backup" && e.action === "settings").pop()!;
|
||||
assert.deepEqual(JSON.parse(entry.detail!).to, { enabled: true, time: "03:30", keep: 14 });
|
||||
});
|
||||
|
||||
test("take one, see it listed, download it byte for byte, check it, and delete it", async () => {
|
||||
const made = await call("POST", "/");
|
||||
assert.equal(made.status, 201);
|
||||
const { snapshot, check } = await made.json();
|
||||
assert.equal(check.ok, true);
|
||||
assert.match(snapshot.name, /-manual\.sqlite$/);
|
||||
|
||||
const overview = await (await call("GET", "/")).json();
|
||||
assert.equal(overview.snapshots.length, 1);
|
||||
assert.equal(overview.snapshots[0].name, snapshot.name);
|
||||
assert.equal(overview.lastAttempt.ok, true);
|
||||
assert.equal(overview.staged, null);
|
||||
|
||||
const download = await call("GET", `/${snapshot.name}/download`);
|
||||
assert.equal(download.status, 200);
|
||||
assert.equal(download.headers.get("cache-control"), "no-store");
|
||||
assert.match(download.headers.get("content-disposition") ?? "", new RegExp(snapshot.name.replace(/\./g, "\\.")));
|
||||
const bytes = Buffer.from(await download.arrayBuffer());
|
||||
assert.deepEqual(bytes, readFileSync(join(sb.backupDir(), snapshot.name)));
|
||||
|
||||
assert.equal((await (await call("POST", `/${snapshot.name}/check`)).json()).check.integrity, "ok");
|
||||
assert.equal((await call("GET", "/homelab-manager-nonsense.sqlite/download")).status, 404);
|
||||
assert.equal((await call("GET", `/..%2Fapp.sqlite/download`)).status, 404);
|
||||
assert.equal((await call("DELETE", `/${snapshot.name}`)).status, 204);
|
||||
assert.equal((await call("DELETE", `/${snapshot.name}`)).status, 404);
|
||||
|
||||
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "db_backup").map((e) => e.action);
|
||||
for (const wanted of ["create", "download", "delete"]) assert.ok(actions.includes(wanted), wanted);
|
||||
});
|
||||
|
||||
test("a backup from elsewhere can be uploaded and then restored like any other; a bad file is refused and leaves nothing", async () => {
|
||||
const { info } = await sb.createSnapshot("manual", at("2026-10-17T18:00:00Z"));
|
||||
const bytes = readFileSync(join(sb.backupDir(), info.name));
|
||||
const before = sb.listSnapshots().length;
|
||||
|
||||
const up = await call("POST", "/upload", undefined, bytes);
|
||||
assert.equal(up.status, 201);
|
||||
const uploaded = (await up.json()).snapshot;
|
||||
assert.match(uploaded.name, /-uploaded\.sqlite$/);
|
||||
assert.equal(sb.listSnapshots().length, before + 1);
|
||||
|
||||
const bad = await call("POST", "/upload", undefined, Buffer.from("not a database"));
|
||||
assert.equal(bad.status, 400);
|
||||
assert.match((await bad.json()).message, /isn't a database file/);
|
||||
assert.equal((await call("POST", "/upload", undefined, Buffer.alloc(0))).status, 400);
|
||||
assert.equal(sb.listSnapshots().length, before + 1, "nothing was kept from the bad ones");
|
||||
|
||||
const restore = await call("POST", `/${uploaded.name}/restore`);
|
||||
assert.equal(restore.status, 200);
|
||||
assert.equal((await restore.json()).staged.from, uploaded.name);
|
||||
assert.equal((await (await call("GET", "/")).json()).staged.from, uploaded.name);
|
||||
assert.equal((await call("POST", "/homelab-manager-2026-01-01-00-00-00.sqlite/restore")).status, 400);
|
||||
assert.equal((await (await call("POST", "/restore/cancel")).json()).cancelled, true);
|
||||
assert.equal((await (await call("GET", "/")).json()).staged, null);
|
||||
assert.ok((await db.select().from(auditLog)).some((e) => e.action === "restore_staged"));
|
||||
assert.ok((await db.select().from(auditLog)).some((e) => e.action === "restore_cancelled"));
|
||||
});
|
||||
|
||||
test("what the last restore did is shown until it is dismissed", async () => {
|
||||
writeFileSync(join(dir, "data", RESULT_NOTE), JSON.stringify({ restoredAt: "2026-10-17T16:00:00.000Z", from: "x.sqlite", keptAs: "y.sqlite", by: "Ada" }));
|
||||
assert.equal((await (await call("GET", "/")).json()).lastRestore.keptAs, "y.sqlite");
|
||||
assert.equal((await call("POST", "/restore-result/dismiss")).status, 200);
|
||||
assert.equal((await (await call("GET", "/")).json()).lastRestore, null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("how current the backups are, in the alerts and /metrics", () => {
|
||||
const selfAlert = async () => {
|
||||
invalidateAlertsCache();
|
||||
return (await getAlerts(true)).alerts.find((a) => a.id === "backup:self-backup");
|
||||
};
|
||||
const oneAged = async (hoursAgo: number) => {
|
||||
wipeBackups();
|
||||
const { info } = await sb.createSnapshot("scheduled", new Date());
|
||||
const when = new Date(Date.now() - hoursAgo * 3600_000);
|
||||
utimesSync(join(sb.backupDir(), info.name), when, when);
|
||||
};
|
||||
|
||||
test("no alert while they are recent, or when none was ever tried; a warning when stale, critical after a week", async () => {
|
||||
wipeBackups();
|
||||
assert.equal(await selfAlert(), undefined, "nothing yet, and nothing failed: a new install isn't scolded");
|
||||
await oneAged(5);
|
||||
assert.equal(await selfAlert(), undefined);
|
||||
await oneAged(60);
|
||||
const stale = await selfAlert();
|
||||
assert.equal(stale!.severity, "warning");
|
||||
assert.match(stale!.message, /60 h old/);
|
||||
assert.equal(stale!.source, "Homelab Manager");
|
||||
assert.equal(stale!.link, "/settings/backup");
|
||||
await oneAged(24 * 8);
|
||||
assert.equal((await selfAlert())!.severity, "critical");
|
||||
assert.match((await selfAlert())!.message, /8 days old/);
|
||||
});
|
||||
|
||||
test("when the last attempt failed, the alert says why; and nothing is said at all while backups are switched off", async () => {
|
||||
await oneAged(60);
|
||||
await setInternalFlag("selfBackupLastAttempt", JSON.stringify({ at: new Date().toISOString(), ok: false, kind: "scheduled", error: "ENOSPC: no space left on device", name: null }));
|
||||
assert.match((await selfAlert())!.message, /The last attempt failed: ENOSPC: no space left on device/);
|
||||
|
||||
wipeBackups();
|
||||
const first = await selfAlert();
|
||||
assert.match(first!.message, /no backup of its own yet\. The last attempt failed|no backup of its own yet/, "a failure with nothing to show for it is said");
|
||||
|
||||
await updateSettings({ selfBackup: { enabled: false } });
|
||||
assert.equal(await selfAlert(), undefined);
|
||||
await updateSettings({ selfBackup: { enabled: true } });
|
||||
});
|
||||
|
||||
test("/metrics says how many are kept, when the newest was made and whether the last try worked", async () => {
|
||||
await oneAged(2);
|
||||
await sb.createSnapshot("manual", new Date());
|
||||
const text = await buildMetrics();
|
||||
assert.match(text, /homelab_selfbackup_snapshots 2/);
|
||||
const ts = /homelab_selfbackup_last_success_timestamp_seconds (\d+)/.exec(text);
|
||||
assert.ok(ts && Math.abs(Number(ts[1]) - Date.now() / 1000) < 120, "the manual one just made is the newest");
|
||||
assert.match(text, /homelab_selfbackup_last_attempt_ok 1/);
|
||||
wipeBackups();
|
||||
});
|
||||
});
|
||||
@@ -282,9 +282,9 @@ describe("following a run to the end", () => {
|
||||
const titles = events.filter((e) => /Update run/.test(e.title)).map((e) => e.title);
|
||||
// (the tests before this one wrote their own lines too, so this looks for the kinds rather than the number)
|
||||
for (const wanted of ["Update run started", "Update run didn't finish", "Update run finished", "Update run stopped being followed"]) assert.ok(titles.includes(wanted), `${wanted} in ${JSON.stringify([...new Set(titles)])}`);
|
||||
const failedLine = events.find((e) => e.title === "Update run didn't finish")!;
|
||||
assert.equal(failedLine.tone, "bad");
|
||||
assert.match(failedLine.detail!, /The playbook failed in Semaphore/);
|
||||
const failedLines = events.filter((e) => e.title === "Update run didn't finish");
|
||||
assert.ok(failedLines.every((e) => e.tone === "bad"), "a run that didn't finish reads as bad");
|
||||
assert.ok(failedLines.some((e) => /The playbook failed in Semaphore/.test(e.detail ?? "")), "and says why");
|
||||
assert.ok((await db.select().from(auditLog)).some((a) => a.category === "update_run" && a.action === "failed"));
|
||||
|
||||
// nothing is held in memory: a fresh pass over the table is all it takes
|
||||
|
||||
Reference in new issue
Block a user