diff --git a/DATABASE.md b/DATABASE.md index 9bb608c..d8c22e8 100644 --- a/DATABASE.md +++ b/DATABASE.md @@ -23,6 +23,7 @@ what the app actually creates. |---|---| | **File** | `DATABASE_PATH`, default `../data/homelab-manager.sqlite` (relative to `server/`; `/app/data/...` in Docker). The folder is created if missing. | | **Foreign keys** | Switched on for every connection (`PRAGMA foreign_keys = ON`), so the `ON DELETE` rules below are enforced. | +| **Writes** | SQLite lets one writer in at a time and refuses a second at once ("database is locked"). While a transaction of the app is open, its other writes wait for it ([`server/src/db/writeGate.ts`](server/src/db/writeGate.ts)) rather than being refused — a refused write also leaves its connection unable to commit the next transaction. Reads are never held; a wait over 10 seconds is given up. A lock held by *another* process (a backup tool reading the file, a second copy of the app) is still refused as before. | | **Migrations** | SQL files in [`server/drizzle/`](server/drizzle) (`0000` … `0082`), applied automatically when the server starts. Which ones have run is recorded in the table `__drizzle_migrations`. | | **Tables** | 91 application tables, plus drizzle's own `__drizzle_migrations`. | diff --git a/server/src/db/client.ts b/server/src/db/client.ts index 3f062c2..e4fb9fe 100644 --- a/server/src/db/client.ts +++ b/server/src/db/client.ts @@ -4,12 +4,15 @@ import { mkdirSync } from "node:fs"; import { dirname } from "node:path"; import { resolveDataPath } from "../paths.js"; import * as schema from "./schema.js"; +import { serialiseWrites } from "./writeGate.js"; const dbPath = resolveDataPath(process.env.DATABASE_PATH ?? "../data/homelab-manager.sqlite"); mkdirSync(dirname(dbPath), { recursive: true }); const client = createClient({ url: `file:${dbPath}` }); await client.execute("PRAGMA foreign_keys = ON;"); +// Ordinary writes wait for a transaction of this process instead of being refused as "database is locked" (see writeGate.ts). +serialiseWrites(client); export const db = drizzle(client, { schema }); export { client, dbPath }; diff --git a/server/src/db/writeGate.ts b/server/src/db/writeGate.ts new file mode 100644 index 0000000..d2b579a --- /dev/null +++ b/server/src/db/writeGate.ts @@ -0,0 +1,100 @@ +import type { Client, Transaction, TransactionMode } from "@libsql/client"; + +/** + * SQLite lets one writer in at a time, and does not queue a second: it is refused at once ("database is locked"). The client keeps one connection for ordinary + * statements and hands another to each open transaction, and a transaction here awaits between its statements, so a write from anywhere else — the diagnostic log + * trimming itself, an agent's report — that lands in one of those pauses is refused. Worse, a refused statement is left half-finished on its connection, and the + * next transaction on that connection then cannot commit ("SQL statements in progress") until the leftover is garbage collected. Trying the write again does not + * help with that, so the writes are kept from being refused instead: while a transaction of this process is open, an ordinary write (and the next transaction) + * waits for it to end. + * + * Reads are never held — SQLite lets them through while a writer is working. A wait that goes on too long (a transaction that was never ended) is given up, and + * the statement goes ahead and is refused as it would have been, rather than the whole app standing still. + */ +export const WRITE_WAIT_LIMIT_MS = 10_000; + +const isRead = (sql: string) => /^\s*(select|pragma|explain)\b/i.test(sql); + +export function serialiseWrites(client: Client, waitLimitMs = WRITE_WAIT_LIMIT_MS): Client { + /** Settles when the transaction that is open now ends; null when none is. */ + let open: Promise | null = null; + const plain = { + execute: client.execute.bind(client), + batch: client.batch.bind(client), + executeMultiple: client.executeMultiple.bind(client), + transaction: client.transaction.bind(client), + }; + + /** + * Runs `run` once no transaction is open. The last look at `open` and the call to `run` are in the same turn of the event loop (no `await` between them), so + * nothing can open a transaction in between. + */ + async function whenQuiet(run: () => Promise): Promise { + const giveUpAt = Date.now() + waitLimitMs; + while (open !== null && Date.now() < giveUpAt) { + let timer: ReturnType | undefined; + const patience = new Promise((resolve) => { + timer = setTimeout(resolve, Math.max(0, giveUpAt - Date.now())); + }); + await Promise.race([open, patience]); + clearTimeout(timer); + } + return run(); + } + + const writing = (call: (...args: A) => Promise) => (...args: A) => whenQuiet(() => call(...args)); + + client.execute = ((...args: Parameters) => { + const [stmt] = args; + const sql = typeof stmt === "string" ? stmt : stmt.sql; + return isRead(sql) ? plain.execute(...args) : whenQuiet(() => plain.execute(...args)); + }) as typeof client.execute; + client.batch = writing(plain.batch) as typeof client.batch; + client.executeMultiple = writing(plain.executeMultiple) as typeof client.executeMultiple; + + client.transaction = ((mode?: TransactionMode) => + whenQuiet(async () => { + // If the wait was given up with another transaction still open, this one is not the one the others wait for. + const gate: { release: () => void } = { release: () => {} }; + const mine = open === null ? new Promise((resolve) => (gate.release = resolve)) : null; + if (mine) open = mine; + const done = () => { + if (mine && open === mine) open = null; + gate.release(); + }; + let tx: Transaction; + try { + tx = await plain.transaction(mode); + } catch (err) { + done(); + throw err; + } + const wrapped: Transaction = { + execute: (stmt) => tx.execute(stmt), + batch: (stmts) => tx.batch(stmts), + executeMultiple: (sql) => tx.executeMultiple(sql), + // A commit that fails leaves the transaction open (the caller rolls it back), so only a commit that went through ends the wait. + commit: async () => { + await tx.commit(); + done(); + }, + rollback: async () => { + try { + await tx.rollback(); + } finally { + done(); + } + }, + close: () => { + tx.close(); + done(); + }, + get closed() { + return tx.closed; + }, + }; + return wrapped; + })) as typeof client.transaction; + + return client; +} diff --git a/server/test/writeGate.test.ts b/server/test/writeGate.test.ts new file mode 100644 index 0000000..197408a --- /dev/null +++ b/server/test/writeGate.test.ts @@ -0,0 +1,165 @@ +/** + * A write that comes while one of the app's own transactions is open waits for it instead of being refused as "database is locked" — which also left the next + * transaction unable to commit. Reads are not held, transactions take turns, and a wait that goes on too long is given up. + * + * Run with: npm test --workspace server + */ +import { test, describe, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const { createClient } = await import("@libsql/client"); +const { drizzle } = await import("drizzle-orm/libsql"); +const { sql } = await import("drizzle-orm"); +const { serialiseWrites } = await import("../src/db/writeGate.js"); + +const dir = mkdtempSync(join(tmpdir(), "hm-writegate-")); +const url = `file:${join(dir, "test.sqlite")}`; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const isBusy = (err: unknown) => String((err as { code?: string }).code).startsWith("SQLITE_BUSY"); + +describe("writes and transactions of one process", () => { + const client = serialiseWrites(createClient({ url }), 1500); + const db = drizzle(client); + const rows = async () => (await client.execute("select x from t order by x")).rows.map((r) => Number(r.x)); + + before(async () => { + await client.execute("create table t (x integer)"); + }); + after(() => { + client.close(); + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // Windows keeps the database file locked until the process exits. + } + }); + + test("a write that comes while a transaction is open waits for it, and neither is refused", async () => { + const tx = await client.transaction("write"); + await tx.execute("insert into t values (1)"); + const started = Date.now(); + const write = client.execute("insert into t values (2)"); + await sleep(250); + await tx.commit(); + await write; + assert.ok(Date.now() - started >= 250, "it waited"); + }); + + test("a refused write used to leave the next transaction unable to commit; now there is none refused", async () => { + const tx = await client.transaction("write"); + await tx.execute("insert into t values (3)"); + const writes = [client.execute("insert into t values (4)"), client.execute("delete from t where x < 0")]; + await sleep(100); + await tx.commit(); + await Promise.all(writes); + const later = await client.transaction("write"); + await later.execute("insert into t values (5)"); + await later.commit(); + assert.deepEqual(await rows(), [1, 2, 3, 4, 5]); + }); + + test("reads are not held by a transaction", async () => { + const tx = await client.transaction("write"); + await tx.execute("insert into t values (6)"); + const started = Date.now(); + assert.deepEqual(await rows(), [1, 2, 3, 4, 5], "what is committed, at once"); + assert.ok(Date.now() - started < 200); + await tx.rollback(); + }); + + test("transactions take turns, and a rollback lets the next one in", async () => { + const order: string[] = []; + const first = await client.transaction("write"); + const second = client.transaction("write").then(async (tx) => { + order.push("second began"); + await tx.execute("insert into t values (8)"); + await tx.commit(); + }); + await first.execute("insert into t values (7)"); + await sleep(150); + order.push("first ends"); + await first.rollback(); + await second; + assert.deepEqual(order, ["first ends", "second began"]); + assert.deepEqual(await rows(), [1, 2, 3, 4, 5, 8]); + }); + + test("drizzle: writes that land in the pauses of a transaction all go through, and the transaction commits", async () => { + const results: Promise[] = []; + await db.transaction(async (tx) => { + for (let i = 0; i < 10; i++) { + await tx.run(sql`insert into t values (${100 + i})`); + results.push(db.run(sql`insert into t values (${200 + i})`)); + await sleep(5); + } + }); + await Promise.all(results); + const all = await rows(); + assert.equal(all.length, 6 + 20); + assert.deepEqual(all.filter((n) => n >= 100 && n < 110).length, 10); + assert.deepEqual(all.filter((n) => n >= 200).length, 10); + }); + + test("a transaction that fails inside rolls back, and the writes held behind it go on", async () => { + const held: Promise[] = []; + await assert.rejects( + db.transaction(async (tx) => { + await tx.run(sql`insert into t values (300)`); + held.push(db.run(sql`insert into t values (301)`)); + await sleep(50); + throw new Error("no good"); + }), + /no good/, + ); + await Promise.all(held); + const all = await rows(); + assert.ok(!all.includes(300), "rolled back"); + assert.ok(all.includes(301), "the write that waited went through"); + }); +}); + +describe("when something else holds the lock, or a transaction is never ended", () => { + const dir2 = mkdtempSync(join(tmpdir(), "hm-writegate2-")); + // Two files: a write that was refused leaves its connection holding the lock until the leftover is cleaned up, which is what the first test makes happen. + const url2 = `file:${join(dir2, "first.sqlite")}`; + const url3 = `file:${join(dir2, "second.sqlite")}`; + after(() => { + try { + rmSync(dir2, { recursive: true, force: true }); + } catch { + // see above + } + }); + + test("a wait that goes on too long is given up: the write is refused as it would have been", async () => { + const client = serialiseWrites(createClient({ url: url2 }), 300); + await client.execute("create table t (x integer)"); + const forgotten = await client.transaction("write"); + await forgotten.execute("insert into t values (1)"); + const started = Date.now(); + await assert.rejects(client.execute("insert into t values (2)"), isBusy); + assert.ok(Date.now() - started >= 250, "it waited first"); + await forgotten.rollback(); + await client.execute("insert into t values (3)"); + assert.deepEqual((await client.execute("select x from t")).rows.map((r) => Number(r.x)), [3]); + client.close(); + }); + + test("a transaction that cannot begin (another process holds the lock) does not hold the others up", async () => { + const client = serialiseWrites(createClient({ url: url3 }), 300); + await client.execute("create table t (x integer)"); + const other = createClient({ url: url3 }); + const theirs = await other.transaction("write"); + await theirs.execute("insert into t values (10)"); + await assert.rejects(client.transaction("write"), isBusy); + await theirs.rollback(); + other.close(); + const started = Date.now(); + await client.execute("insert into t values (11)"); + assert.ok(Date.now() - started < 200, "no wait left behind"); + client.close(); + }); +});