Acknowledge and hand over alerts, incident timelines, and built-in database backups
- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics - Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents, incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs, related alerts and notes; Incidents page and a digest section - Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download, check, upload, delete, and restore staged and applied at start-up before the database opens (pre-restore copy kept); alerts and /metrics for stale or failing backups - Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7a5b623444
commit
2744a84719
41 files changed
+6573
-26
No files matched your search
@@ -0,0 +1,323 @@
|
||||
/**
|
||||
* Backups of Homelab Manager itself. Everything it knows is one SQLite file, so a backup is a consistent copy of that file (`VACUUM INTO`,
|
||||
* which is safe while the app is working) kept in the data folder's `backups/`: taken every day, the newest few kept, each one checked as
|
||||
* it is made, and downloadable, uploadable and restorable from Settings → Backup.
|
||||
*
|
||||
* What a backup is not: it does not hold `CREDENTIALS_ENCRYPTION_KEY`, which decrypts the credentials stored in it, and it isn't off the
|
||||
* machine until someone copies it there. Both are said where backups are shown.
|
||||
*
|
||||
* A restore can't swap a database that is open, so it is staged (checked, then copied beside the database) and takes effect when the app
|
||||
* next starts — see db/applyRestore.ts, which does it before anything opens the file.
|
||||
*/
|
||||
import { createClient } from "@libsql/client";
|
||||
import { copyFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, openSync, readdirSync, readFileSync, readSync, closeSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { client, dbPath } from "../db/client.js";
|
||||
import { PENDING_FILE, PENDING_NOTE, RESULT_NOTE, type RestoreResult } from "../db/restoreFiles.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export type SnapshotKind = "scheduled" | "manual" | "pre-restore" | "uploaded";
|
||||
|
||||
export interface SnapshotInfo {
|
||||
name: string;
|
||||
kind: SnapshotKind;
|
||||
sizeBytes: number;
|
||||
/** ISO — when the file was written. */
|
||||
at: string;
|
||||
}
|
||||
|
||||
export interface SnapshotCheck {
|
||||
ok: boolean;
|
||||
/** SQLite's own verdict ("ok", or what it found). */
|
||||
integrity: string | null;
|
||||
/** How many schema changes it has had; goes up with each release that changes the database. */
|
||||
migrations: number | null;
|
||||
/** How many this version of the app knows. */
|
||||
appMigrations: number;
|
||||
/** Made by a newer version of the app than this one: it can't be restored here. */
|
||||
newerThanApp: boolean;
|
||||
counts: { users: number; servers: number; integrations: number } | null;
|
||||
/** Why it isn't usable, when it isn't. */
|
||||
problem: string | null;
|
||||
}
|
||||
|
||||
const NAME = /^homelab-manager-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}(?:-(manual|pre-restore|uploaded))?(?:-\d+)?\.sqlite$/;
|
||||
export const isSnapshotName = (name: string) => NAME.test(name);
|
||||
|
||||
const dataDir = () => dirname(dbPath);
|
||||
export const backupDir = () => join(dataDir(), "backups");
|
||||
|
||||
const stamp = (d: Date) => d.toISOString().slice(0, 19).replace(/[:T]/g, "-");
|
||||
|
||||
function kindOf(name: string): SnapshotKind {
|
||||
const m = NAME.exec(name);
|
||||
return (m?.[1] as SnapshotKind | undefined) ?? "scheduled";
|
||||
}
|
||||
|
||||
/** How many schema changes this version of the app has (the entries in its migration journal). */
|
||||
export function appMigrationCount(): number {
|
||||
try {
|
||||
const journal = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "drizzle", "meta", "_journal.json"), "utf8")) as { entries?: unknown[] };
|
||||
return journal.entries?.length ?? 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function hasSqliteHeader(file: string): boolean {
|
||||
try {
|
||||
const fd = openSync(file, "r");
|
||||
try {
|
||||
const buf = Buffer.alloc(16);
|
||||
readSync(fd, buf, 0, 16, 0);
|
||||
return buf.toString("latin1") === "SQLite format 3\0";
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** A scratch folder for looking at a copy of a backup, and the way to be rid of it (best effort: some systems hold an opened database until the process ends). */
|
||||
function scratch(): { file: string; done: () => void } {
|
||||
const folder = mkdtempSync(join(tmpdir(), "hlm-backup-"));
|
||||
return {
|
||||
file: join(folder, "db.sqlite"),
|
||||
done: () => {
|
||||
try {
|
||||
rmSync(folder, { recursive: true, force: true });
|
||||
} catch {
|
||||
// left for the system's own tidying of its temporary folder
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Looks at a backup: does SQLite think it is sound, is it one of ours, and is it from this version or an older one? It looks at a copy in a
|
||||
* temporary folder, never at the file itself, so a backup in the backups folder is never held open — it can always be deleted, anywhere.
|
||||
*/
|
||||
export async function checkSnapshotFile(file: string): Promise<SnapshotCheck> {
|
||||
if (!existsSync(file)) return checkOpened(file);
|
||||
if (!hasSqliteHeader(file)) return checkOpened(file);
|
||||
const tmp = scratch();
|
||||
try {
|
||||
copyFileSync(file, tmp.file);
|
||||
return await checkOpened(tmp.file);
|
||||
} finally {
|
||||
tmp.done();
|
||||
}
|
||||
}
|
||||
|
||||
/** The check itself, on a file nobody else cares about the handle of. */
|
||||
async function checkOpened(file: string): Promise<SnapshotCheck> {
|
||||
const appMigrations = appMigrationCount();
|
||||
const fail = (problem: string, extra: Partial<SnapshotCheck> = {}): SnapshotCheck => ({ ok: false, integrity: null, migrations: null, appMigrations, newerThanApp: false, counts: null, problem, ...extra });
|
||||
if (!existsSync(file)) return fail("The file isn't there.");
|
||||
if (!hasSqliteHeader(file)) return fail("That isn't a database file.");
|
||||
const c = createClient({ url: `file:${file}` });
|
||||
try {
|
||||
const one = async (sql: string) => Object.values((await c.execute(sql)).rows[0] ?? {})[0];
|
||||
const integrity = String(await one("PRAGMA integrity_check"));
|
||||
const tables = new Set((await c.execute("select name from sqlite_master where type = 'table'")).rows.map((r) => String(r.name)));
|
||||
for (const needed of ["users", "servers", "integrations", "settings", "__drizzle_migrations"]) {
|
||||
if (!tables.has(needed)) return fail(`It isn't a Homelab Manager database (no "${needed}" table).`, { integrity });
|
||||
}
|
||||
const migrations = Number(await one("select count(*) from __drizzle_migrations"));
|
||||
const counts = { users: Number(await one("select count(*) from users")), servers: Number(await one("select count(*) from servers")), integrations: Number(await one("select count(*) from integrations")) };
|
||||
const newerThanApp = appMigrations > 0 && migrations > appMigrations;
|
||||
if (integrity !== "ok") return fail(`SQLite found problems in it: ${integrity}`, { integrity, migrations, counts, newerThanApp });
|
||||
if (newerThanApp) return fail("It was made by a newer version of Homelab Manager than this one, and an older version can't use it. Update first.", { integrity, migrations, counts, newerThanApp });
|
||||
return { ok: true, integrity, migrations, appMigrations, newerThanApp: false, counts, problem: null };
|
||||
} catch (err) {
|
||||
return fail(`It couldn't be read as a database: ${err instanceof Error ? err.message : String(err)}`);
|
||||
} finally {
|
||||
c.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** A name for a new file that isn't taken. */
|
||||
function freshName(kind: SnapshotKind, now: Date): string {
|
||||
const base = `homelab-manager-${stamp(now)}${kind === "scheduled" ? "" : `-${kind}`}`;
|
||||
let name = `${base}.sqlite`;
|
||||
for (let n = 2; existsSync(join(backupDir(), name)); n++) name = `${base}-${n}.sqlite`;
|
||||
return name;
|
||||
}
|
||||
|
||||
export interface LastAttempt {
|
||||
at: string;
|
||||
ok: boolean;
|
||||
kind: SnapshotKind;
|
||||
error: string | null;
|
||||
name: string | null;
|
||||
}
|
||||
const ATTEMPT_FLAG = "selfBackupLastAttempt";
|
||||
|
||||
export async function lastAttempt(): Promise<LastAttempt | null> {
|
||||
try {
|
||||
const v = await getInternalFlag(ATTEMPT_FLAG);
|
||||
return v ? (JSON.parse(v) as LastAttempt) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes a consistent copy of the database into the backups folder and checks it. Throws, saying why, if it can't — and records the attempt either way. */
|
||||
export async function createSnapshot(kind: SnapshotKind = "manual", now: Date = new Date()): Promise<{ info: SnapshotInfo; check: SnapshotCheck }> {
|
||||
const tmp = scratch();
|
||||
try {
|
||||
mkdirSync(backupDir(), { recursive: true });
|
||||
const name = freshName(kind, now);
|
||||
const file = join(backupDir(), name);
|
||||
// Made and checked in a temporary folder first, and only then put in the backups folder: what is there is always whole and sound.
|
||||
await client.execute(`VACUUM INTO '${tmp.file.replace(/'/g, "''")}'`);
|
||||
const check = await checkOpened(tmp.file);
|
||||
if (!check.ok) throw new Error(`The copy didn't check out, so it was thrown away: ${check.problem}`);
|
||||
copyFileSync(tmp.file, file);
|
||||
try {
|
||||
chmodSync(file, 0o600);
|
||||
} catch {
|
||||
// not every file system has modes
|
||||
}
|
||||
const info: SnapshotInfo = { name, kind, sizeBytes: statSync(file).size, at: now.toISOString() };
|
||||
await setInternalFlag(ATTEMPT_FLAG, JSON.stringify({ at: now.toISOString(), ok: true, kind, error: null, name } satisfies LastAttempt));
|
||||
return { info, check };
|
||||
} catch (err) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
await setInternalFlag(ATTEMPT_FLAG, JSON.stringify({ at: now.toISOString(), ok: false, kind, error, name: null } satisfies LastAttempt));
|
||||
throw err;
|
||||
} finally {
|
||||
tmp.done();
|
||||
}
|
||||
}
|
||||
|
||||
export function listSnapshots(): SnapshotInfo[] {
|
||||
if (!existsSync(backupDir())) return [];
|
||||
return readdirSync(backupDir())
|
||||
.filter(isSnapshotName)
|
||||
.map((name) => {
|
||||
const st = statSync(join(backupDir(), name));
|
||||
return { name, kind: kindOf(name), sizeBytes: st.size, at: st.mtime.toISOString() };
|
||||
})
|
||||
.sort((a, b) => b.at.localeCompare(a.at) || b.name.localeCompare(a.name));
|
||||
}
|
||||
|
||||
/** What is kept: the newest `keep` scheduled backups, and a few of each other kind (a restore's "before" copy is worth keeping longer than a day). */
|
||||
export const KEEP_OTHER: Record<Exclude<SnapshotKind, "scheduled">, number> = { manual: 10, "pre-restore": 3, uploaded: 3 };
|
||||
|
||||
export function pruneSnapshots(keep: number): string[] {
|
||||
const removed: string[] = [];
|
||||
const all = listSnapshots();
|
||||
const limitFor = (kind: SnapshotKind) => (kind === "scheduled" ? Math.max(1, keep) : KEEP_OTHER[kind]);
|
||||
for (const kind of ["scheduled", "manual", "pre-restore", "uploaded"] as const) {
|
||||
for (const s of all.filter((x) => x.kind === kind).slice(limitFor(kind))) {
|
||||
rmSync(join(backupDir(), s.name), { force: true });
|
||||
removed.push(s.name);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
export function snapshotPath(name: string): string | null {
|
||||
if (!isSnapshotName(name)) return null;
|
||||
const file = join(backupDir(), name);
|
||||
return existsSync(file) ? file : null;
|
||||
}
|
||||
|
||||
export function deleteSnapshot(name: string): boolean {
|
||||
const file = snapshotPath(name);
|
||||
if (!file) return false;
|
||||
rmSync(file, { force: true });
|
||||
return true;
|
||||
}
|
||||
|
||||
/** A file somebody uploaded, kept as a backup if it checks out (and thrown away if not). */
|
||||
export async function importUploaded(bytes: Buffer, now: Date = new Date()): Promise<{ info: SnapshotInfo; check: SnapshotCheck }> {
|
||||
const tmp = scratch();
|
||||
try {
|
||||
writeFileSync(tmp.file, bytes);
|
||||
const check = await checkOpened(tmp.file);
|
||||
if (!check.ok) throw new Error(check.problem ?? "That file isn't usable.");
|
||||
mkdirSync(backupDir(), { recursive: true });
|
||||
const name = freshName("uploaded", now);
|
||||
const file = join(backupDir(), name);
|
||||
copyFileSync(tmp.file, file);
|
||||
try {
|
||||
chmodSync(file, 0o600);
|
||||
} catch {
|
||||
// not every file system has modes
|
||||
}
|
||||
return { info: { name, kind: "uploaded", sizeBytes: statSync(file).size, at: now.toISOString() }, check };
|
||||
} finally {
|
||||
tmp.done();
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Restoring ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface StagedRestore {
|
||||
from: string;
|
||||
stagedAt: string;
|
||||
by: string | null;
|
||||
}
|
||||
|
||||
export function stagedRestore(): StagedRestore | null {
|
||||
try {
|
||||
if (!existsSync(join(dataDir(), PENDING_FILE))) return null;
|
||||
const note = JSON.parse(readFileSync(join(dataDir(), PENDING_NOTE), "utf8")) as { from?: string; stagedAt?: string; by?: string | null };
|
||||
return { from: note.from ?? "a backup", stagedAt: note.stagedAt ?? "", by: note.by ?? null };
|
||||
} catch {
|
||||
return existsSync(join(dataDir(), PENDING_FILE)) ? { from: "a backup", stagedAt: "", by: null } : null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Checks a backup and stages it to replace the database the next time the app starts. Throws, saying why, if it can't be used. */
|
||||
export async function stageRestore(name: string, by: string | null, now: Date = new Date()): Promise<SnapshotCheck> {
|
||||
const file = snapshotPath(name);
|
||||
if (!file) throw new Error("There is no backup with that name.");
|
||||
const check = await checkSnapshotFile(file);
|
||||
if (!check.ok) throw new Error(check.problem ?? "That backup isn't usable.");
|
||||
const pending = join(dataDir(), PENDING_FILE);
|
||||
const tmp = `${pending}.copying`;
|
||||
copyFileSync(file, tmp);
|
||||
renameSync(tmp, pending); // whole, or not there at all: the start-up step never sees half a file
|
||||
try {
|
||||
chmodSync(pending, 0o600);
|
||||
} catch {
|
||||
// not every file system has modes
|
||||
}
|
||||
writeFileSync(join(dataDir(), PENDING_NOTE), JSON.stringify({ from: name, stagedAt: now.toISOString(), by }));
|
||||
return check;
|
||||
}
|
||||
|
||||
export function cancelStagedRestore(): boolean {
|
||||
const had = existsSync(join(dataDir(), PENDING_FILE));
|
||||
rmSync(join(dataDir(), PENDING_FILE), { force: true });
|
||||
rmSync(join(dataDir(), PENDING_NOTE), { force: true });
|
||||
return had;
|
||||
}
|
||||
|
||||
/** What the last restore did, once, for the page to say. */
|
||||
export function lastRestore(): RestoreResult | null {
|
||||
try {
|
||||
return JSON.parse(readFileSync(join(dataDir(), RESULT_NOTE), "utf8")) as RestoreResult;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function dismissLastRestore(): void {
|
||||
rmSync(join(dataDir(), RESULT_NOTE), { force: true });
|
||||
}
|
||||
|
||||
// ─── How current the backups are ────────────────────────────────────────────
|
||||
|
||||
/** Backups older than this are an alert: a day's slack on top of the daily schedule. */
|
||||
export const STALE_AFTER_HOURS = 50;
|
||||
|
||||
export function newestSnapshot(): SnapshotInfo | null {
|
||||
return listSnapshots().find((s) => s.kind === "scheduled" || s.kind === "manual") ?? null;
|
||||
}
|
||||
Reference in new issue
Block a user