Acknowledge and hand over alerts, incident timelines, and built-in database backups
- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics - Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents, incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs, related alerts and notes; Incidents page and a digest section - Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download, check, upload, delete, and restore staged and applied at start-up before the database opens (pre-restore copy kept); alerts and /metrics for stale or failing backups - Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7a5b623444
commit
2744a84719
41 files changed
+6573
-26
No files matched your search
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* Finishes a restore that was staged from Settings → Backup. It has to happen before the database is opened, which is why it is a module
|
||||
* of its own that index.ts imports first: by the time anything else loads, the file is already the one that was chosen.
|
||||
*
|
||||
* Nothing is deleted. The database that was in use is moved into the backups folder (as "…-pre-restore.sqlite") before the chosen one is put
|
||||
* in its place, so a restore that wasn't what was wanted can itself be undone by restoring that file. If anything goes wrong half way, the
|
||||
* old file is put back and the staged one is left where it is.
|
||||
*/
|
||||
import "../loadEnv.js";
|
||||
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { resolveDataPath } from "../paths.js";
|
||||
|
||||
import { PENDING_FILE, PENDING_NOTE, RESULT_NOTE, type RestoreResult } from "./restoreFiles.js";
|
||||
|
||||
const stamp = (d: Date) => d.toISOString().slice(0, 19).replace(/[:T]/g, "-");
|
||||
|
||||
/** Puts a staged database in place, if one is waiting. Returns what it did, or null if nothing was staged. */
|
||||
export function applyPendingRestore(dbFile: string, now: Date = new Date()): RestoreResult | null {
|
||||
const dataDir = dirname(dbFile);
|
||||
const pending = join(dataDir, PENDING_FILE);
|
||||
if (!existsSync(pending)) return null;
|
||||
// Only a file is ever put in place of the database.
|
||||
if (!statSync(pending).isFile()) {
|
||||
console.error(`[restore] ${PENDING_FILE} isn't a file, so it was left alone and the database is as it was.`);
|
||||
return null;
|
||||
}
|
||||
|
||||
let note: { from?: string; by?: string | null } = {};
|
||||
try {
|
||||
note = JSON.parse(readFileSync(join(dataDir, PENDING_NOTE), "utf8"));
|
||||
} catch {
|
||||
// the note only says where it came from; the file is what matters
|
||||
}
|
||||
|
||||
const backups = join(dataDir, "backups");
|
||||
let keptAs: string | null = null;
|
||||
let moved = false;
|
||||
try {
|
||||
mkdirSync(backups, { recursive: true });
|
||||
if (existsSync(dbFile)) {
|
||||
let name = `homelab-manager-${stamp(now)}-pre-restore.sqlite`;
|
||||
for (let n = 2; existsSync(join(backups, name)); n++) name = `homelab-manager-${stamp(now)}-pre-restore-${n}.sqlite`;
|
||||
renameSync(dbFile, join(backups, name));
|
||||
keptAs = name;
|
||||
moved = true;
|
||||
}
|
||||
// A journal left by the old file would be taken for the new one's.
|
||||
for (const suffix of ["-wal", "-shm", "-journal"]) rmSync(dbFile + suffix, { force: true });
|
||||
renameSync(pending, dbFile);
|
||||
} catch (err) {
|
||||
if (moved && keptAs && !existsSync(dbFile)) {
|
||||
try {
|
||||
renameSync(join(backups, keptAs), dbFile);
|
||||
} catch {
|
||||
// nothing more can be done; the old file is still in the backups folder under its new name
|
||||
}
|
||||
}
|
||||
console.error("[restore] couldn't put the staged backup in place; the database is as it was:", err instanceof Error ? err.message : err);
|
||||
return null;
|
||||
}
|
||||
|
||||
const result: RestoreResult = { restoredAt: now.toISOString(), from: note.from ?? "an uploaded backup", keptAs, by: note.by ?? null };
|
||||
writeFileSync(join(dataDir, RESULT_NOTE), JSON.stringify(result));
|
||||
rmSync(join(dataDir, PENDING_NOTE), { force: true });
|
||||
console.log(`[restore] restored the database from ${result.from}${keptAs ? `; the one that was in use is kept as backups/${keptAs}` : ""}.`);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Run it now, on the database this process is about to open.
|
||||
applyPendingRestore(resolveDataPath(process.env.DATABASE_PATH ?? "../data/homelab-manager.sqlite"));
|
||||
@@ -12,4 +12,4 @@ const client = createClient({ url: `file:${dbPath}` });
|
||||
await client.execute("PRAGMA foreign_keys = ON;");
|
||||
|
||||
export const db = drizzle(client, { schema });
|
||||
export { client };
|
||||
export { client, dbPath };
|
||||
@@ -0,0 +1,13 @@
|
||||
// The names a staged restore uses in the data folder. Kept apart from applyRestore.ts, which does something when it is imported.
|
||||
export const PENDING_FILE = "restore-pending.sqlite";
|
||||
export const PENDING_NOTE = "restore-pending.json";
|
||||
export const RESULT_NOTE = "restore-result.json";
|
||||
|
||||
export interface RestoreResult {
|
||||
restoredAt: string;
|
||||
/** The backup that is now the database. */
|
||||
from: string;
|
||||
/** Where the database that was in use went. */
|
||||
keptAs: string | null;
|
||||
by: string | null;
|
||||
}
|
||||
@@ -572,6 +572,63 @@ export const runbooks = sqliteTable(
|
||||
(t) => [index("runbooks_target_idx").on(t.targetType, t.targetId)],
|
||||
);
|
||||
|
||||
// ─── Acknowledgements and incidents ─────────────────────────────────────────
|
||||
|
||||
// "I'm on it" for one alert — see services/alertAcks.ts. Keyed by the alert's stable id (not a foreign key to anything); gone once the alert has been.
|
||||
export const alertAcks = sqliteTable(
|
||||
"alert_acks",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
alertKey: text("alert_key").notNull(),
|
||||
note: text("note").notNull().default(""),
|
||||
assigneeUserId: integer("assignee_user_id").references(() => users.id, { onDelete: "set null" }),
|
||||
assigneeName: text("assignee_name"), // as it was when handed over, so it still reads if the user is removed
|
||||
ackedBy: text("acked_by").notNull(),
|
||||
ackedAt: text("acked_at").notNull(), // ISO
|
||||
until: text("until"), // ISO; null = until the alert is gone
|
||||
lastSeenAt: text("last_seen_at").notNull(), // ISO; the last time the alert was there
|
||||
},
|
||||
(t) => [uniqueIndex("alert_acks_key_idx").on(t.alertKey)],
|
||||
);
|
||||
|
||||
// One time an alert was raised and later went away — see services/incidents.ts. When the same alert comes back it is a new row.
|
||||
export const alertIncidents = sqliteTable(
|
||||
"alert_incidents",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
alertKey: text("alert_key").notNull(),
|
||||
severity: text("severity").notNull(), // as it is now (or was when it ended)
|
||||
peakSeverity: text("peak_severity").notNull(),
|
||||
category: text("category").notNull(),
|
||||
source: text("source").notNull(),
|
||||
message: text("message").notNull(), // the latest wording
|
||||
link: text("link"),
|
||||
targetType: text("target_type"), // server | integration | dns_provider, when the alert is about one of those
|
||||
targetId: integer("target_id"),
|
||||
targetName: text("target_name"),
|
||||
causedBy: text("caused_by"), // the host whose outage this was folded under
|
||||
openedAt: text("opened_at").notNull(), // ISO
|
||||
resolvedAt: text("resolved_at"), // ISO; null while it is open
|
||||
lastSeenAt: text("last_seen_at").notNull(), // ISO
|
||||
missingSince: text("missing_since"), // ISO; set while it has been absent from the last look(s), before it is called over
|
||||
},
|
||||
(t) => [index("alert_incidents_key_idx").on(t.alertKey), index("alert_incidents_opened_idx").on(t.openedAt), index("alert_incidents_target_idx").on(t.targetType, t.targetId)],
|
||||
);
|
||||
|
||||
export const incidentNotes = sqliteTable(
|
||||
"incident_notes",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
incidentId: integer("incident_id")
|
||||
.notNull()
|
||||
.references(() => alertIncidents.id, { onDelete: "cascade" }),
|
||||
author: text("author").notNull(),
|
||||
body: text("body").notNull(),
|
||||
createdAt: text("created_at").notNull(), // ISO
|
||||
},
|
||||
(t) => [index("incident_notes_incident_idx").on(t.incidentId)],
|
||||
);
|
||||
|
||||
// ─── Update runs: apply a server's updates and see it through ──────────────
|
||||
|
||||
// A playbook run in Semaphore for one server, wrapped in a maintenance window and followed until the server reports again — see
|
||||
|
||||
Reference in new issue
Block a user