/** * Backups of Homelab Manager itself: taking them, checking them, keeping the newest, staging and applying a restore, the routes around * them, and what the alerts and /metrics say about how current they are. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; import type { Server } from "node:http"; const dir = mkdtempSync(join(tmpdir(), "hm-selfbackup-")); process.env.DATABASE_PATH = join(dir, "data", "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { createClient } = await import("@libsql/client"); const { db, client } = await import("../src/db/client.js"); const { users, auditLog } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const sb = await import("../src/services/selfBackup.js"); const { runScheduledBackup, cronForBackup } = await import("../src/services/selfBackupScheduler.js"); const { applyPendingRestore } = await import("../src/db/applyRestore.js"); const { PENDING_FILE, PENDING_NOTE, RESULT_NOTE } = await import("../src/db/restoreFiles.js"); const { dbBackupsRouter } = await import("../src/routes/dbBackups.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { buildMetrics } = await import("../src/services/metrics.js"); const { getSettings, updateSettings, setInternalFlag } = await import("../src/services/settingsStore.js"); const schedule = (await import("node-schedule")).default; await runMigrations(); after(async () => { await schedule.gracefulShutdown(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); const at = (iso: string) => new Date(iso); /** Opens a copy of a backup, never the backup itself: an opened database can't be deleted on some systems until the process ends. */ const open = (file: string) => { const copy = join(mkdtempSync(join(dir, "copy-")), "db.sqlite"); copyFileSync(file, copy); return createClient({ url: `file:${copy}` }); }; const wipeBackups = () => rmSync(sb.backupDir(), { recursive: true, force: true }); describe("taking and checking a backup", () => { test("a backup is a consistent copy that opens, passes SQLite's check, and holds what was there", async () => { await db.insert(users).values({ oidcSub: "ada", name: "Ada", role: "admin" }); const { info, check } = await sb.createSnapshot("manual", at("2026-10-17T12:00:00Z")); assert.equal(info.name, "homelab-manager-2026-10-17-12-00-00-manual.sqlite"); assert.equal(info.kind, "manual"); assert.ok(info.sizeBytes > 10_000); assert.equal(check.ok, true); assert.equal(check.integrity, "ok"); assert.equal(check.migrations, sb.appMigrationCount()); assert.equal(check.counts!.users, 1); // and it really holds it: open the file on its own and read the row back const copy = open(join(sb.backupDir(), info.name)); const rows = (await copy.execute("select name from users")).rows; copy.close(); assert.deepEqual(rows.map((r) => r.name), ["Ada"]); }); test("changes made after it was taken are not in it", async () => { const { info } = await sb.createSnapshot("manual", at("2026-10-17T12:01:00Z")); await db.insert(users).values({ oidcSub: "bo", name: "Bo", role: "viewer" }); const copy = open(join(sb.backupDir(), info.name)); assert.equal(Number((await copy.execute("select count(*) as n from users")).rows[0].n), 1); copy.close(); await db.delete(users).where((await import("drizzle-orm")).eq(users.oidcSub, "bo")); }); test("two taken in the same second don't collide", async () => { const a = await sb.createSnapshot("manual", at("2026-10-17T13:00:00Z")); const b = await sb.createSnapshot("manual", at("2026-10-17T13:00:00Z")); assert.notEqual(a.info.name, b.info.name); assert.match(b.info.name, /-manual-2\.sqlite$/); }); test("files that aren't a usable backup are said to be, and why", async () => { const junk = join(dir, "junk.sqlite"); writeFileSync(junk, "this is not a database at all"); assert.match((await sb.checkSnapshotFile(junk)).problem!, /isn't a database file/); assert.match((await sb.checkSnapshotFile(join(dir, "missing.sqlite"))).problem!, /isn't there/); // a real SQLite file that isn't ours const other = join(dir, "other.sqlite"); const c = createClient({ url: `file:${other}` }); await c.execute("create table cats (name text)"); c.close(); assert.match((await sb.checkSnapshotFile(other)).problem!, /isn't a Homelab Manager database/); // one of ours, made by a newer version (more schema changes than this one has) const newer = join(dir, "newer.sqlite"); await client.execute(`VACUUM INTO '${newer.replace(/'/g, "''")}'`); const n = createClient({ url: `file:${newer}` }); for (let i = 0; i < 3; i++) await n.execute({ sql: "insert into __drizzle_migrations (hash, created_at) values (?, ?)", args: [`future-${i}`, 9_999_999_999_999 + i] }); n.close(); const check = await sb.checkSnapshotFile(newer); assert.equal(check.ok, false); assert.equal(check.newerThanApp, true); assert.match(check.problem!, /newer version/); assert.ok(check.migrations! > check.appMigrations); }); test("an older one is fine: this version brings it up to date when it starts", async () => { const older = join(dir, "older.sqlite"); await client.execute(`VACUUM INTO '${older.replace(/'/g, "''")}'`); const o = createClient({ url: `file:${older}` }); await o.execute("delete from __drizzle_migrations where rowid = (select max(rowid) from __drizzle_migrations)"); o.close(); const check = await sb.checkSnapshotFile(older); assert.equal(check.ok, true); assert.equal(check.migrations, sb.appMigrationCount() - 1); }); test("only names of our own files are ever acted on, whatever is asked for", () => { for (const bad of ["../test.sqlite", "..\\test.sqlite", "homelab-manager-2026-10-17-12-00-00.sqlite/../../x", "homelab-manager-x.sqlite", "test.sqlite", "homelab-manager-2026-10-17-12-00-00.sqlite.bak", "/etc/passwd", ""]) { assert.equal(sb.isSnapshotName(bad), false, bad); assert.equal(sb.snapshotPath(bad), null, bad); assert.equal(sb.deleteSnapshot(bad), false, bad); } for (const good of ["homelab-manager-2026-10-17-12-00-00.sqlite", "homelab-manager-2026-10-17-12-00-00-manual.sqlite", "homelab-manager-2026-10-17-12-00-00-pre-restore-2.sqlite", "homelab-manager-2026-10-17-12-00-00-uploaded.sqlite"]) { assert.equal(sb.isSnapshotName(good), true, good); } }); }); describe("keeping the newest", () => { const fake = (name: string, mtime: string) => { mkdirSync(sb.backupDir(), { recursive: true }); const f = join(sb.backupDir(), name); writeFileSync(f, "x"); utimesSync(f, at(mtime), at(mtime)); }; const names = () => sb.listSnapshots().map((s) => s.name); test("the newest N of the daily ones stay, and each other kind has its own, smaller allowance", () => { wipeBackups(); for (let d = 1; d <= 6; d++) fake(`homelab-manager-2026-10-0${d}-03-30-00.sqlite`, `2026-10-0${d}T03:30:00Z`); for (let d = 1; d <= 12; d++) fake(`homelab-manager-2026-10-${String(d).padStart(2, "0")}-09-00-00-manual.sqlite`, `2026-10-${String(d).padStart(2, "0")}T09:00:00Z`); for (let d = 1; d <= 5; d++) fake(`homelab-manager-2026-10-0${d}-10-00-00-pre-restore.sqlite`, `2026-10-0${d}T10:00:00Z`); const removed = sb.pruneSnapshots(3); const left = names(); assert.equal(left.filter((n) => !/-(manual|pre-restore|uploaded)/.test(n)).length, 3); assert.equal(left.filter((n) => n.includes("-manual")).length, 10); assert.equal(left.filter((n) => n.includes("-pre-restore")).length, 3); assert.equal(removed.length, 3 + 2 + 2); assert.ok(left.includes("homelab-manager-2026-10-06-03-30-00.sqlite") && !left.includes("homelab-manager-2026-10-01-03-30-00.sqlite"), "the oldest went, not the newest"); // other files in the folder are left alone writeFileSync(join(sb.backupDir(), "notes.txt"), "mine"); sb.pruneSnapshots(1); assert.ok(existsSync(join(sb.backupDir(), "notes.txt"))); wipeBackups(); }); test("a scheduled run takes one and trims to what is kept, and records that it worked", async () => { wipeBackups(); await updateSettings({ selfBackup: { keep: 2 } }); for (const d of ["2026-10-14T03:30:00Z", "2026-10-15T03:30:00Z", "2026-10-16T03:30:00Z"]) assert.ok(await runScheduledBackup(at(d))); assert.deepEqual(names(), ["homelab-manager-2026-10-16-03-30-00.sqlite", "homelab-manager-2026-10-15-03-30-00.sqlite"]); const last = await sb.lastAttempt(); assert.equal(last!.ok, true); assert.equal(last!.kind, "scheduled"); assert.equal(cronForBackup("03:30"), "30 3 * * *"); assert.equal(cronForBackup("23:05"), "5 23 * * *"); await updateSettings({ selfBackup: { keep: 14 } }); }); test("a backup that can't be made is recorded as having failed, says why, and leaves no half file", async () => { wipeBackups(); const realExecute = client.execute.bind(client); (client as unknown as { execute: unknown }).execute = async (stmt: unknown) => { if (typeof stmt === "string" && stmt.startsWith("VACUUM INTO")) throw new Error("SQLITE_FULL: database or disk is full"); return realExecute(stmt as never); }; try { await assert.rejects(sb.createSnapshot("scheduled", at("2026-10-17T03:30:00Z")), /SQLITE_FULL/); const last = await sb.lastAttempt(); assert.equal(last!.ok, false); assert.match(last!.error!, /disk is full/); assert.deepEqual(sb.listSnapshots(), [], "nothing half-written is left in the backups folder"); assert.equal(await runScheduledBackup(at("2026-10-17T03:31:00Z")), null, "the scheduled run swallows it (and says so through the channels)"); } finally { (client as unknown as { execute: unknown }).execute = realExecute; } assert.ok(await runScheduledBackup(at("2026-10-17T03:32:00Z")), "and the next one works"); assert.equal((await sb.lastAttempt())!.ok, true); wipeBackups(); }); }); describe("restoring", () => { test("a staged restore is checked first, kept beside the database, can be cancelled, and refuses what isn't usable", async () => { wipeBackups(); const { info } = await sb.createSnapshot("manual", at("2026-10-17T15:00:00Z")); assert.equal(sb.stagedRestore(), null); await assert.rejects(sb.stageRestore("../../etc/passwd", "Ada"), /no backup with that name/); await assert.rejects(sb.stageRestore("homelab-manager-2026-01-01-00-00-00.sqlite", "Ada"), /no backup with that name/); writeFileSync(join(sb.backupDir(), "homelab-manager-2026-10-17-16-00-00-uploaded.sqlite"), "garbage"); await assert.rejects(sb.stageRestore("homelab-manager-2026-10-17-16-00-00-uploaded.sqlite", "Ada"), /isn't a database file/); assert.equal(sb.stagedRestore(), null, "a refused one stages nothing"); const check = await sb.stageRestore(info.name, "Ada", at("2026-10-17T15:05:00Z")); assert.equal(check.ok, true); const staged = sb.stagedRestore()!; assert.deepEqual(staged, { from: info.name, stagedAt: "2026-10-17T15:05:00.000Z", by: "Ada" }); const dataDir = join(dir, "data"); assert.ok(existsSync(join(dataDir, PENDING_FILE))); assert.equal(readdirSync(dataDir).some((f) => f.endsWith(".copying")), false, "no half-copied file is left"); assert.equal(statSync(join(dataDir, PENDING_FILE)).size, info.sizeBytes); assert.equal(sb.cancelStagedRestore(), true); assert.equal(sb.stagedRestore(), null); assert.equal(existsSync(join(dataDir, PENDING_FILE)), false); assert.equal(sb.cancelStagedRestore(), false); wipeBackups(); }); test("at start-up the staged file replaces the database, the old one is kept, and nothing happens twice", () => { const folder = mkdtempSync(join(dir, "apply-")); const dbFile = join(folder, "app.sqlite"); writeFileSync(dbFile, "OLD DATABASE"); writeFileSync(`${dbFile}-wal`, "stale journal"); writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE"); writeFileSync(join(folder, PENDING_NOTE), JSON.stringify({ from: "homelab-manager-2026-10-01-03-30-00.sqlite", by: "Ada" })); const result = applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z"))!; assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE"); assert.equal(existsSync(`${dbFile}-wal`), false, "a journal from the old file would be taken for the new one's"); assert.equal(result.keptAs, "homelab-manager-2026-10-17-16-00-00-pre-restore.sqlite"); assert.equal(readFileSync(join(folder, "backups", result.keptAs!), "utf8"), "OLD DATABASE", "what was in use is kept, not deleted"); assert.equal(result.from, "homelab-manager-2026-10-01-03-30-00.sqlite"); assert.equal(result.by, "Ada"); assert.equal(existsSync(join(folder, PENDING_FILE)), false); assert.equal(existsSync(join(folder, PENDING_NOTE)), false); assert.deepEqual(JSON.parse(readFileSync(join(folder, RESULT_NOTE), "utf8")), result); assert.equal(applyPendingRestore(dbFile, at("2026-10-17T16:01:00Z")), null, "nothing staged any more"); assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE"); }); test("with no database there yet (a fresh machine), the staged one simply becomes it", () => { const folder = mkdtempSync(join(dir, "fresh-")); const dbFile = join(folder, "app.sqlite"); writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE"); const result = applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z"))!; assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE"); assert.equal(result.keptAs, null); assert.equal(result.from, "an uploaded backup", "no note, no name — said plainly"); }); test("a restore that can't be completed leaves the database exactly as it was", () => { // the folder for keeping the old one can't be made (a file is in its place) const folder = mkdtempSync(join(dir, "fail-")); const dbFile = join(folder, "app.sqlite"); writeFileSync(dbFile, "OLD DATABASE"); writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE"); writeFileSync(join(folder, "backups"), "I am a file, not a folder"); assert.equal(applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z")), null); assert.equal(readFileSync(dbFile, "utf8"), "OLD DATABASE", "the database is as it was"); assert.equal(readFileSync(join(folder, PENDING_FILE), "utf8"), "NEW DATABASE", "and the staged one is still there to try again"); assert.equal(existsSync(join(folder, RESULT_NOTE)), false); // something staged that isn't a file is never put in the database's place const odd = mkdtempSync(join(dir, "odd-")); writeFileSync(join(odd, "app.sqlite"), "OLD DATABASE"); mkdirSync(join(odd, PENDING_FILE)); assert.equal(applyPendingRestore(join(odd, "app.sqlite"), at("2026-10-17T16:00:00Z")), null); assert.equal(readFileSync(join(odd, "app.sqlite"), "utf8"), "OLD DATABASE"); }); }); describe("through the routes", () => { let app: Server; let base = ""; let who = "ada"; const call = (method: string, path: string, body?: unknown, raw?: Buffer) => fetch(`${base}${path}`, raw ? { method, headers: { "content-type": "application/octet-stream" }, body: new Uint8Array(raw) } : { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); before(async () => { await db.insert(users).values({ oidcSub: "operator", name: "Olle", role: "operator" }); const a = express(); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/db-backups", dbBackupsRouter); app = await new Promise((resolve) => { const x = a.listen(0, "127.0.0.1", () => resolve(x)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/db-backups`; wipeBackups(); }); after(() => app.close()); test("everything about backups is for administrators only", async () => { who = "operator"; for (const [method, path] of [["GET", "/"], ["POST", "/"], ["PUT", "/settings"], ["POST", "/upload"], ["GET", "/x/download"], ["POST", "/x/restore"], ["POST", "/x/check"], ["DELETE", "/x"], ["POST", "/restore/cancel"]] as const) { assert.equal((await call(method, path, method === "GET" || method === "DELETE" ? undefined : {})).status, 403, `${method} ${path}`); } who = "ada"; assert.equal((await call("GET", "/")).status, 200); }); test("settings are checked and kept, and change what is kept at once", async () => { assert.equal((await call("PUT", "/settings", { enabled: true, time: "25:00", keep: 7 })).status, 400); assert.equal((await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 0 })).status, 400); assert.equal((await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 500 })).status, 400); const res = await call("PUT", "/settings", { enabled: true, time: "04:15", keep: 7 }); assert.equal(res.status, 200); const body = await res.json(); assert.deepEqual(body.settings, { enabled: true, time: "04:15", keep: 7 }); assert.ok(body.nextRun, "it says when the next one is"); assert.equal((await getSettings()).selfBackup.time, "04:15"); const off = await (await call("PUT", "/settings", { enabled: false, time: "04:15", keep: 7 })).json(); assert.equal(off.nextRun, null, "off means nothing is scheduled"); await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 14 }); const entry = (await db.select().from(auditLog)).filter((e) => e.category === "db_backup" && e.action === "settings").pop()!; assert.deepEqual(JSON.parse(entry.detail!).to, { enabled: true, time: "03:30", keep: 14 }); }); test("take one, see it listed, download it byte for byte, check it, and delete it", async () => { const made = await call("POST", "/"); assert.equal(made.status, 201); const { snapshot, check } = await made.json(); assert.equal(check.ok, true); assert.match(snapshot.name, /-manual\.sqlite$/); const overview = await (await call("GET", "/")).json(); assert.equal(overview.snapshots.length, 1); assert.equal(overview.snapshots[0].name, snapshot.name); assert.equal(overview.lastAttempt.ok, true); assert.equal(overview.staged, null); const download = await call("GET", `/${snapshot.name}/download`); assert.equal(download.status, 200); assert.equal(download.headers.get("cache-control"), "no-store"); assert.match(download.headers.get("content-disposition") ?? "", new RegExp(snapshot.name.replace(/\./g, "\\."))); const bytes = Buffer.from(await download.arrayBuffer()); assert.deepEqual(bytes, readFileSync(join(sb.backupDir(), snapshot.name))); assert.equal((await (await call("POST", `/${snapshot.name}/check`)).json()).check.integrity, "ok"); assert.equal((await call("GET", "/homelab-manager-nonsense.sqlite/download")).status, 404); assert.equal((await call("GET", `/..%2Fapp.sqlite/download`)).status, 404); assert.equal((await call("DELETE", `/${snapshot.name}`)).status, 204); assert.equal((await call("DELETE", `/${snapshot.name}`)).status, 404); const actions = (await db.select().from(auditLog)).filter((e) => e.category === "db_backup").map((e) => e.action); for (const wanted of ["create", "download", "delete"]) assert.ok(actions.includes(wanted), wanted); }); test("a backup from elsewhere can be uploaded and then restored like any other; a bad file is refused and leaves nothing", async () => { const { info } = await sb.createSnapshot("manual", at("2026-10-17T18:00:00Z")); const bytes = readFileSync(join(sb.backupDir(), info.name)); const before = sb.listSnapshots().length; const up = await call("POST", "/upload", undefined, bytes); assert.equal(up.status, 201); const uploaded = (await up.json()).snapshot; assert.match(uploaded.name, /-uploaded\.sqlite$/); assert.equal(sb.listSnapshots().length, before + 1); const bad = await call("POST", "/upload", undefined, Buffer.from("not a database")); assert.equal(bad.status, 400); assert.match((await bad.json()).message, /isn't a database file/); assert.equal((await call("POST", "/upload", undefined, Buffer.alloc(0))).status, 400); assert.equal(sb.listSnapshots().length, before + 1, "nothing was kept from the bad ones"); const restore = await call("POST", `/${uploaded.name}/restore`); assert.equal(restore.status, 200); assert.equal((await restore.json()).staged.from, uploaded.name); assert.equal((await (await call("GET", "/")).json()).staged.from, uploaded.name); assert.equal((await call("POST", "/homelab-manager-2026-01-01-00-00-00.sqlite/restore")).status, 400); assert.equal((await (await call("POST", "/restore/cancel")).json()).cancelled, true); assert.equal((await (await call("GET", "/")).json()).staged, null); assert.ok((await db.select().from(auditLog)).some((e) => e.action === "restore_staged")); assert.ok((await db.select().from(auditLog)).some((e) => e.action === "restore_cancelled")); }); test("what the last restore did is shown until it is dismissed", async () => { writeFileSync(join(dir, "data", RESULT_NOTE), JSON.stringify({ restoredAt: "2026-10-17T16:00:00.000Z", from: "x.sqlite", keptAs: "y.sqlite", by: "Ada" })); assert.equal((await (await call("GET", "/")).json()).lastRestore.keptAs, "y.sqlite"); assert.equal((await call("POST", "/restore-result/dismiss")).status, 200); assert.equal((await (await call("GET", "/")).json()).lastRestore, null); }); }); describe("how current the backups are, in the alerts and /metrics", () => { const selfAlert = async () => { invalidateAlertsCache(); return (await getAlerts(true)).alerts.find((a) => a.id === "backup:self-backup"); }; const oneAged = async (hoursAgo: number) => { wipeBackups(); const { info } = await sb.createSnapshot("scheduled", new Date()); const when = new Date(Date.now() - hoursAgo * 3600_000); utimesSync(join(sb.backupDir(), info.name), when, when); }; test("no alert while they are recent, or when none was ever tried; a warning when stale, critical after a week", async () => { wipeBackups(); assert.equal(await selfAlert(), undefined, "nothing yet, and nothing failed: a new install isn't scolded"); await oneAged(5); assert.equal(await selfAlert(), undefined); await oneAged(60); const stale = await selfAlert(); assert.equal(stale!.severity, "warning"); assert.match(stale!.message, /60 h old/); assert.equal(stale!.source, "Homelab Manager"); assert.equal(stale!.link, "/settings/backup"); await oneAged(24 * 8); assert.equal((await selfAlert())!.severity, "critical"); assert.match((await selfAlert())!.message, /8 days old/); }); test("when the last attempt failed, the alert says why; and nothing is said at all while backups are switched off", async () => { await oneAged(60); await setInternalFlag("selfBackupLastAttempt", JSON.stringify({ at: new Date().toISOString(), ok: false, kind: "scheduled", error: "ENOSPC: no space left on device", name: null })); assert.match((await selfAlert())!.message, /The last attempt failed: ENOSPC: no space left on device/); wipeBackups(); const first = await selfAlert(); assert.match(first!.message, /no backup of its own yet\. The last attempt failed|no backup of its own yet/, "a failure with nothing to show for it is said"); await updateSettings({ selfBackup: { enabled: false } }); assert.equal(await selfAlert(), undefined); await updateSettings({ selfBackup: { enabled: true } }); }); test("/metrics says how many are kept, when the newest was made and whether the last try worked", async () => { await oneAged(2); await sb.createSnapshot("manual", new Date()); const text = await buildMetrics(); assert.match(text, /homelab_selfbackup_snapshots 2/); const ts = /homelab_selfbackup_last_success_timestamp_seconds (\d+)/.exec(text); assert.ok(ts && Math.abs(Number(ts[1]) - Date.now() / 1000) < 120, "the manual one just made is the newest"); assert.match(text, /homelab_selfbackup_last_attempt_ok 1/); wipeBackups(); }); });