Keep the app's own writes from being refused as locked while one of its transactions is open
SQLite lets one writer in at a time and refuses a second at once. A write that landed in the pauses of an in-process transaction (the diagnostic log trimming itself, an agent report) was refused with SQLITE_BUSY, and the refused statement left the next transaction unable to commit. Ordinary writes and the next transaction now wait for the open transaction; reads are never held, and a wait over 10 s is given up. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c79b2eda0d
commit
c7775e5dd9
4 files changed
+269
No files matched your search
@@ -23,6 +23,7 @@ what the app actually creates.
|
|||||||
|---|---|
|
|---|---|
|
||||||
| **File** | `DATABASE_PATH`, default `../data/homelab-manager.sqlite` (relative to `server/`; `/app/data/...` in Docker). The folder is created if missing. |
|
| **File** | `DATABASE_PATH`, default `../data/homelab-manager.sqlite` (relative to `server/`; `/app/data/...` in Docker). The folder is created if missing. |
|
||||||
| **Foreign keys** | Switched on for every connection (`PRAGMA foreign_keys = ON`), so the `ON DELETE` rules below are enforced. |
|
| **Foreign keys** | Switched on for every connection (`PRAGMA foreign_keys = ON`), so the `ON DELETE` rules below are enforced. |
|
||||||
|
| **Writes** | SQLite lets one writer in at a time and refuses a second at once ("database is locked"). While a transaction of the app is open, its other writes wait for it ([`server/src/db/writeGate.ts`](server/src/db/writeGate.ts)) rather than being refused — a refused write also leaves its connection unable to commit the next transaction. Reads are never held; a wait over 10 seconds is given up. A lock held by *another* process (a backup tool reading the file, a second copy of the app) is still refused as before. |
|
||||||
| **Migrations** | SQL files in [`server/drizzle/`](server/drizzle) (`0000` … `0082`), applied automatically when the server starts. Which ones have run is recorded in the table `__drizzle_migrations`. |
|
| **Migrations** | SQL files in [`server/drizzle/`](server/drizzle) (`0000` … `0082`), applied automatically when the server starts. Which ones have run is recorded in the table `__drizzle_migrations`. |
|
||||||
| **Tables** | 91 application tables, plus drizzle's own `__drizzle_migrations`. |
|
| **Tables** | 91 application tables, plus drizzle's own `__drizzle_migrations`. |
|
||||||
|
|
||||||
|
|||||||
@@ -4,12 +4,15 @@ import { mkdirSync } from "node:fs";
|
|||||||
import { dirname } from "node:path";
|
import { dirname } from "node:path";
|
||||||
import { resolveDataPath } from "../paths.js";
|
import { resolveDataPath } from "../paths.js";
|
||||||
import * as schema from "./schema.js";
|
import * as schema from "./schema.js";
|
||||||
|
import { serialiseWrites } from "./writeGate.js";
|
||||||
|
|
||||||
const dbPath = resolveDataPath(process.env.DATABASE_PATH ?? "../data/homelab-manager.sqlite");
|
const dbPath = resolveDataPath(process.env.DATABASE_PATH ?? "../data/homelab-manager.sqlite");
|
||||||
mkdirSync(dirname(dbPath), { recursive: true });
|
mkdirSync(dirname(dbPath), { recursive: true });
|
||||||
|
|
||||||
const client = createClient({ url: `file:${dbPath}` });
|
const client = createClient({ url: `file:${dbPath}` });
|
||||||
await client.execute("PRAGMA foreign_keys = ON;");
|
await client.execute("PRAGMA foreign_keys = ON;");
|
||||||
|
// Ordinary writes wait for a transaction of this process instead of being refused as "database is locked" (see writeGate.ts).
|
||||||
|
serialiseWrites(client);
|
||||||
|
|
||||||
export const db = drizzle(client, { schema });
|
export const db = drizzle(client, { schema });
|
||||||
export { client, dbPath };
|
export { client, dbPath };
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
import type { Client, Transaction, TransactionMode } from "@libsql/client";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SQLite lets one writer in at a time, and does not queue a second: it is refused at once ("database is locked"). The client keeps one connection for ordinary
|
||||||
|
* statements and hands another to each open transaction, and a transaction here awaits between its statements, so a write from anywhere else — the diagnostic log
|
||||||
|
* trimming itself, an agent's report — that lands in one of those pauses is refused. Worse, a refused statement is left half-finished on its connection, and the
|
||||||
|
* next transaction on that connection then cannot commit ("SQL statements in progress") until the leftover is garbage collected. Trying the write again does not
|
||||||
|
* help with that, so the writes are kept from being refused instead: while a transaction of this process is open, an ordinary write (and the next transaction)
|
||||||
|
* waits for it to end.
|
||||||
|
*
|
||||||
|
* Reads are never held — SQLite lets them through while a writer is working. A wait that goes on too long (a transaction that was never ended) is given up, and
|
||||||
|
* the statement goes ahead and is refused as it would have been, rather than the whole app standing still.
|
||||||
|
*/
|
||||||
|
export const WRITE_WAIT_LIMIT_MS = 10_000;
|
||||||
|
|
||||||
|
const isRead = (sql: string) => /^\s*(select|pragma|explain)\b/i.test(sql);
|
||||||
|
|
||||||
|
export function serialiseWrites(client: Client, waitLimitMs = WRITE_WAIT_LIMIT_MS): Client {
|
||||||
|
/** Settles when the transaction that is open now ends; null when none is. */
|
||||||
|
let open: Promise<void> | null = null;
|
||||||
|
const plain = {
|
||||||
|
execute: client.execute.bind(client),
|
||||||
|
batch: client.batch.bind(client),
|
||||||
|
executeMultiple: client.executeMultiple.bind(client),
|
||||||
|
transaction: client.transaction.bind(client),
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Runs `run` once no transaction is open. The last look at `open` and the call to `run` are in the same turn of the event loop (no `await` between them), so
|
||||||
|
* nothing can open a transaction in between.
|
||||||
|
*/
|
||||||
|
async function whenQuiet<T>(run: () => Promise<T>): Promise<T> {
|
||||||
|
const giveUpAt = Date.now() + waitLimitMs;
|
||||||
|
while (open !== null && Date.now() < giveUpAt) {
|
||||||
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
const patience = new Promise<void>((resolve) => {
|
||||||
|
timer = setTimeout(resolve, Math.max(0, giveUpAt - Date.now()));
|
||||||
|
});
|
||||||
|
await Promise.race([open, patience]);
|
||||||
|
clearTimeout(timer);
|
||||||
|
}
|
||||||
|
return run();
|
||||||
|
}
|
||||||
|
|
||||||
|
const writing = <A extends unknown[], R>(call: (...args: A) => Promise<R>) => (...args: A) => whenQuiet(() => call(...args));
|
||||||
|
|
||||||
|
client.execute = ((...args: Parameters<typeof plain.execute>) => {
|
||||||
|
const [stmt] = args;
|
||||||
|
const sql = typeof stmt === "string" ? stmt : stmt.sql;
|
||||||
|
return isRead(sql) ? plain.execute(...args) : whenQuiet(() => plain.execute(...args));
|
||||||
|
}) as typeof client.execute;
|
||||||
|
client.batch = writing(plain.batch) as typeof client.batch;
|
||||||
|
client.executeMultiple = writing(plain.executeMultiple) as typeof client.executeMultiple;
|
||||||
|
|
||||||
|
client.transaction = ((mode?: TransactionMode) =>
|
||||||
|
whenQuiet(async () => {
|
||||||
|
// If the wait was given up with another transaction still open, this one is not the one the others wait for.
|
||||||
|
const gate: { release: () => void } = { release: () => {} };
|
||||||
|
const mine = open === null ? new Promise<void>((resolve) => (gate.release = resolve)) : null;
|
||||||
|
if (mine) open = mine;
|
||||||
|
const done = () => {
|
||||||
|
if (mine && open === mine) open = null;
|
||||||
|
gate.release();
|
||||||
|
};
|
||||||
|
let tx: Transaction;
|
||||||
|
try {
|
||||||
|
tx = await plain.transaction(mode);
|
||||||
|
} catch (err) {
|
||||||
|
done();
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
const wrapped: Transaction = {
|
||||||
|
execute: (stmt) => tx.execute(stmt),
|
||||||
|
batch: (stmts) => tx.batch(stmts),
|
||||||
|
executeMultiple: (sql) => tx.executeMultiple(sql),
|
||||||
|
// A commit that fails leaves the transaction open (the caller rolls it back), so only a commit that went through ends the wait.
|
||||||
|
commit: async () => {
|
||||||
|
await tx.commit();
|
||||||
|
done();
|
||||||
|
},
|
||||||
|
rollback: async () => {
|
||||||
|
try {
|
||||||
|
await tx.rollback();
|
||||||
|
} finally {
|
||||||
|
done();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
close: () => {
|
||||||
|
tx.close();
|
||||||
|
done();
|
||||||
|
},
|
||||||
|
get closed() {
|
||||||
|
return tx.closed;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return wrapped;
|
||||||
|
})) as typeof client.transaction;
|
||||||
|
|
||||||
|
return client;
|
||||||
|
}
|
||||||
@@ -0,0 +1,165 @@
|
|||||||
|
/**
|
||||||
|
* A write that comes while one of the app's own transactions is open waits for it instead of being refused as "database is locked" — which also left the next
|
||||||
|
* transaction unable to commit. Reads are not held, transactions take turns, and a wait that goes on too long is given up.
|
||||||
|
*
|
||||||
|
* Run with: npm test --workspace server
|
||||||
|
*/
|
||||||
|
import { test, describe, before, after } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { mkdtempSync, rmSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
const { createClient } = await import("@libsql/client");
|
||||||
|
const { drizzle } = await import("drizzle-orm/libsql");
|
||||||
|
const { sql } = await import("drizzle-orm");
|
||||||
|
const { serialiseWrites } = await import("../src/db/writeGate.js");
|
||||||
|
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), "hm-writegate-"));
|
||||||
|
const url = `file:${join(dir, "test.sqlite")}`;
|
||||||
|
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
const isBusy = (err: unknown) => String((err as { code?: string }).code).startsWith("SQLITE_BUSY");
|
||||||
|
|
||||||
|
describe("writes and transactions of one process", () => {
|
||||||
|
const client = serialiseWrites(createClient({ url }), 1500);
|
||||||
|
const db = drizzle(client);
|
||||||
|
const rows = async () => (await client.execute("select x from t order by x")).rows.map((r) => Number(r.x));
|
||||||
|
|
||||||
|
before(async () => {
|
||||||
|
await client.execute("create table t (x integer)");
|
||||||
|
});
|
||||||
|
after(() => {
|
||||||
|
client.close();
|
||||||
|
try {
|
||||||
|
rmSync(dir, { recursive: true, force: true });
|
||||||
|
} catch {
|
||||||
|
// Windows keeps the database file locked until the process exits.
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a write that comes while a transaction is open waits for it, and neither is refused", async () => {
|
||||||
|
const tx = await client.transaction("write");
|
||||||
|
await tx.execute("insert into t values (1)");
|
||||||
|
const started = Date.now();
|
||||||
|
const write = client.execute("insert into t values (2)");
|
||||||
|
await sleep(250);
|
||||||
|
await tx.commit();
|
||||||
|
await write;
|
||||||
|
assert.ok(Date.now() - started >= 250, "it waited");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a refused write used to leave the next transaction unable to commit; now there is none refused", async () => {
|
||||||
|
const tx = await client.transaction("write");
|
||||||
|
await tx.execute("insert into t values (3)");
|
||||||
|
const writes = [client.execute("insert into t values (4)"), client.execute("delete from t where x < 0")];
|
||||||
|
await sleep(100);
|
||||||
|
await tx.commit();
|
||||||
|
await Promise.all(writes);
|
||||||
|
const later = await client.transaction("write");
|
||||||
|
await later.execute("insert into t values (5)");
|
||||||
|
await later.commit();
|
||||||
|
assert.deepEqual(await rows(), [1, 2, 3, 4, 5]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("reads are not held by a transaction", async () => {
|
||||||
|
const tx = await client.transaction("write");
|
||||||
|
await tx.execute("insert into t values (6)");
|
||||||
|
const started = Date.now();
|
||||||
|
assert.deepEqual(await rows(), [1, 2, 3, 4, 5], "what is committed, at once");
|
||||||
|
assert.ok(Date.now() - started < 200);
|
||||||
|
await tx.rollback();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("transactions take turns, and a rollback lets the next one in", async () => {
|
||||||
|
const order: string[] = [];
|
||||||
|
const first = await client.transaction("write");
|
||||||
|
const second = client.transaction("write").then(async (tx) => {
|
||||||
|
order.push("second began");
|
||||||
|
await tx.execute("insert into t values (8)");
|
||||||
|
await tx.commit();
|
||||||
|
});
|
||||||
|
await first.execute("insert into t values (7)");
|
||||||
|
await sleep(150);
|
||||||
|
order.push("first ends");
|
||||||
|
await first.rollback();
|
||||||
|
await second;
|
||||||
|
assert.deepEqual(order, ["first ends", "second began"]);
|
||||||
|
assert.deepEqual(await rows(), [1, 2, 3, 4, 5, 8]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("drizzle: writes that land in the pauses of a transaction all go through, and the transaction commits", async () => {
|
||||||
|
const results: Promise<unknown>[] = [];
|
||||||
|
await db.transaction(async (tx) => {
|
||||||
|
for (let i = 0; i < 10; i++) {
|
||||||
|
await tx.run(sql`insert into t values (${100 + i})`);
|
||||||
|
results.push(db.run(sql`insert into t values (${200 + i})`));
|
||||||
|
await sleep(5);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
await Promise.all(results);
|
||||||
|
const all = await rows();
|
||||||
|
assert.equal(all.length, 6 + 20);
|
||||||
|
assert.deepEqual(all.filter((n) => n >= 100 && n < 110).length, 10);
|
||||||
|
assert.deepEqual(all.filter((n) => n >= 200).length, 10);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a transaction that fails inside rolls back, and the writes held behind it go on", async () => {
|
||||||
|
const held: Promise<unknown>[] = [];
|
||||||
|
await assert.rejects(
|
||||||
|
db.transaction(async (tx) => {
|
||||||
|
await tx.run(sql`insert into t values (300)`);
|
||||||
|
held.push(db.run(sql`insert into t values (301)`));
|
||||||
|
await sleep(50);
|
||||||
|
throw new Error("no good");
|
||||||
|
}),
|
||||||
|
/no good/,
|
||||||
|
);
|
||||||
|
await Promise.all(held);
|
||||||
|
const all = await rows();
|
||||||
|
assert.ok(!all.includes(300), "rolled back");
|
||||||
|
assert.ok(all.includes(301), "the write that waited went through");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("when something else holds the lock, or a transaction is never ended", () => {
|
||||||
|
const dir2 = mkdtempSync(join(tmpdir(), "hm-writegate2-"));
|
||||||
|
// Two files: a write that was refused leaves its connection holding the lock until the leftover is cleaned up, which is what the first test makes happen.
|
||||||
|
const url2 = `file:${join(dir2, "first.sqlite")}`;
|
||||||
|
const url3 = `file:${join(dir2, "second.sqlite")}`;
|
||||||
|
after(() => {
|
||||||
|
try {
|
||||||
|
rmSync(dir2, { recursive: true, force: true });
|
||||||
|
} catch {
|
||||||
|
// see above
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a wait that goes on too long is given up: the write is refused as it would have been", async () => {
|
||||||
|
const client = serialiseWrites(createClient({ url: url2 }), 300);
|
||||||
|
await client.execute("create table t (x integer)");
|
||||||
|
const forgotten = await client.transaction("write");
|
||||||
|
await forgotten.execute("insert into t values (1)");
|
||||||
|
const started = Date.now();
|
||||||
|
await assert.rejects(client.execute("insert into t values (2)"), isBusy);
|
||||||
|
assert.ok(Date.now() - started >= 250, "it waited first");
|
||||||
|
await forgotten.rollback();
|
||||||
|
await client.execute("insert into t values (3)");
|
||||||
|
assert.deepEqual((await client.execute("select x from t")).rows.map((r) => Number(r.x)), [3]);
|
||||||
|
client.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a transaction that cannot begin (another process holds the lock) does not hold the others up", async () => {
|
||||||
|
const client = serialiseWrites(createClient({ url: url3 }), 300);
|
||||||
|
await client.execute("create table t (x integer)");
|
||||||
|
const other = createClient({ url: url3 });
|
||||||
|
const theirs = await other.transaction("write");
|
||||||
|
await theirs.execute("insert into t values (10)");
|
||||||
|
await assert.rejects(client.transaction("write"), isBusy);
|
||||||
|
await theirs.rollback();
|
||||||
|
other.close();
|
||||||
|
const started = Date.now();
|
||||||
|
await client.execute("insert into t values (11)");
|
||||||
|
assert.ok(Date.now() - started < 200, "no wait left behind");
|
||||||
|
client.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in new issue
Block a user