Files
bobbanandClaude Sonnet 5.5 251a4a9d73 Prometheus, Immich and Paperless-ngx integrations
Three read-only integrations, each with a monitoring page, alerts, maintenance
silencing and held-when-unreadable handling.

Prometheus: scrape targets that stop answering (5 minutes, one alert per job)
and alerts that Prometheus's own rules fire (critical when labelled so).
Immich: disk under the library filling, failed background jobs, counts.
Paperless-ngx: worker/database/Redis in error, documents that failed to
import, index and sanity check, disk. Immich and Paperless also in /metrics.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-24 22:33:17 +02:00

295 lines
17 KiB
TypeScript

/**
* Paperless-ngx: reading what its API says (status components, statistics, failed imports), what is stored, what is wrong, the health pass, the alerts, the page's route and /metrics.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-paperless-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, paperlessState } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/paperless/adapter.js");
const pl = await import("../src/services/paperlessSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { paperlessRouter } = await import("../src/routes/paperless.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const DAY = 86_400_000;
const NOW = new Date("2026-10-20T12:00:00Z");
type Read = import("../src/integrations/paperless/adapter.js").PaperlessRead;
const comp = (name: string, status = "OK", error: string | null = null) => ({ name, status, error });
const read = (extra: Partial<Read> = {}): Read => ({
version: "2.14.7", latestVersion: "2.14.7", updateAvailable: false, storage: { total: 1e12, available: 6e11 },
components: [comp("database"), comp("redis"), comp("celery"), comp("index"), comp("classifier"), comp("sanity_check")], documents: 4200, inbox: 3, failed: [], ...extra,
});
const STATUS = {
pngx_version: "2.14.7",
storage: { total: 1e12, available: 6e11 },
database: { type: "sqlite", status: "OK", error: null },
tasks: { redis_status: "OK", celery_status: "ERROR", celery_error: "No celery workers responded", index_status: "OK", classifier_status: "WARNING", classifier_error: "Classifier has not been trained yet", sanity_check_status: "OK" },
};
describe("what Paperless's API says", () => {
test("the system status: the version, the disk and each component with how it is", () => {
const s = adapter.parseStatus(STATUS);
assert.equal(s.version, "2.14.7");
assert.deepEqual(s.storage, { total: 1e12, available: 6e11 });
assert.deepEqual(s.components?.map((c) => [c.name, c.status, c.error]), [["database", "OK", null], ["redis", "OK", null], ["celery", "ERROR", "No celery workers responded"], ["index", "OK", null], ["classifier", "WARNING", "Classifier has not been trained yet"], ["sanity_check", "OK", null]]);
assert.deepEqual(adapter.parseStatus({}), { version: null, storage: null, components: null });
});
test("the documents and the inbox", () => {
assert.deepEqual(adapter.parseStatistics({ documents_total: 4200, documents_inbox: 3 }), { documents: 4200, inbox: 3 });
assert.deepEqual(adapter.parseStatistics(null), { documents: null, inbox: null });
});
test("failed imports: only failures, not dismissed, from the last week, newest first; an answer without a list is not 'none'", () => {
const f = adapter.parseFailures(
[
{ status: "FAILURE", task_file_name: "old.pdf", date_done: new Date(NOW.getTime() - 10 * DAY).toISOString(), result: "x" },
{ status: "FAILURE", task_file_name: "a.pdf", date_done: new Date(NOW.getTime() - 2 * DAY).toISOString(), result: "Not a valid PDF" },
{ status: "FAILURE", task_file_name: "dismissed.pdf", acknowledged: true, date_done: NOW.toISOString() },
{ status: "SUCCESS", task_file_name: "fine.pdf", date_done: NOW.toISOString() },
{ status: "FAILURE", task_file_name: "b.pdf", date_done: new Date(NOW.getTime() - 3600_000).toISOString() },
],
NOW,
)!;
assert.deepEqual(f.map((x) => [x.file, x.error]), [["b.pdf", null], ["a.pdf", "Not a valid PDF"]]);
assert.equal(adapter.parseFailures({ results: [] }, NOW)?.length, 0, "a paginated answer is read too");
assert.equal(adapter.parseFailures({ detail: "no" }, NOW), null);
});
test("a newer version", () => {
assert.deepEqual(adapter.parseRemoteVersion({ version: "v2.15.0", update_available: true }), { latestVersion: "v2.15.0", updateAvailable: true });
assert.deepEqual(adapter.parseRemoteVersion(null), { latestVersion: null, updateAvailable: false });
});
describe("the adapter against a Paperless that answers", () => {
let server: Server;
let url = "";
let superuser = true;
const seen: (string | undefined)[] = [];
before(async () => {
server = createHttpServer((req, res) => {
seen.push(req.headers.authorization);
const send = (code: number, body: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(body));
};
if (req.headers.authorization !== "Token tok") return send(401, {});
const path = (req.url ?? "").split("?")[0];
if (path === "/api/statistics/") return send(200, { documents_total: 100, documents_inbox: 2 });
if (path === "/api/status/") return superuser ? send(200, STATUS) : send(403, {});
if (path === "/api/tasks/") return send(200, [{ status: "FAILURE", task_file_name: "x.pdf", date_done: new Date().toISOString(), result: "boom" }]);
if (path === "/api/remote_version/") return send(200, { version: "2.15.0", update_available: true });
send(404, {});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
});
after(() => server.close());
test("it reads everything and sends the token", async () => {
const r = await adapter.createPaperlessAdapter({ url: `${url}/`, token: "tok" }).read(new Date());
assert.deepEqual([r.version, r.documents, r.inbox, r.updateAvailable, r.latestVersion], ["2.14.7", 100, 2, true, "2.15.0"]);
assert.deepEqual(r.failed?.map((f) => f.file), ["x.pdf"]);
assert.equal(r.components?.find((c) => c.name === "celery")?.status, "ERROR");
assert.ok(seen.every((s) => s === "Token tok"));
});
test("a token that is not a superuser's still reads the rest; the status is unknown, not fine", async () => {
superuser = false;
const r = await adapter.createPaperlessAdapter({ url, token: "tok" }).read(new Date());
assert.deepEqual([r.components, r.storage, r.version], [null, null, null]);
assert.equal(r.documents, 100);
});
test("a refused token and a dead address are said plainly", async () => {
const bad = await adapter.createPaperlessAdapter({ url, token: "nope" }).ping();
assert.equal(bad.ok, false);
assert.match(bad.error ?? "", /did not accept the token/);
assert.equal((await adapter.createPaperlessAdapter({ url, token: "tok" }).ping()).ok, true);
assert.equal((await adapter.createPaperlessAdapter({ url: "http://127.0.0.1:1", token: "tok" }).ping()).ok, false);
});
});
});
describe("what is stored", () => {
let id = 0;
before(async () => {
id = (await db.insert(integrations).values({ type: "paperless", name: "Documents", baseUrl: "http://10.9.9.1:8000", config: JSON.stringify({ url: "http://10.9.9.1:8000", token: "tok" }) }).returning({ id: integrations.id }))[0].id;
});
const state = async () => (await db.select().from(paperlessState).where(eq(paperlessState.integrationId, id)))[0];
test("the state is replaced at each read; what could not be read stays null, not empty", async () => {
await pl.storeRead(id, read({ failed: [{ file: "a.pdf", error: null, at: null }] }), NOW);
assert.deepEqual(pl.failedOf(await state())?.map((f) => f.file), ["a.pdf"]);
await pl.storeRead(id, read({ components: null, failed: null, storage: null }), NOW);
const s = await state();
assert.deepEqual([pl.componentsOf(s), pl.failedOf(s), s.storageTotal], [null, null, null]);
});
test("a read that failed is said, and what was stored is left alone", async () => {
const ok = await pl.syncPaperless(id, new Date(), async () => read({ version: "2.15.0" }));
assert.equal(ok.ok, true);
const failed = await pl.syncPaperless(id, new Date(), async () => {
throw new Error("connection refused");
});
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
assert.equal(failed.at, ok.at);
assert.equal((await state()).version, "2.15.0");
assert.equal(pl.isPaperlessFresh((await pl.paperlessSyncStatuses())[id], Date.now()), false);
});
});
describe("what is wrong", () => {
const row = (extra: { storageTotal?: number | null; storageAvailable?: number | null; components?: unknown; failed?: unknown } = {}) => ({
storageTotal: 1e12 as number | null,
storageAvailable: 6e11 as number | null,
components: JSON.stringify(extra.components ?? [comp("database"), comp("celery")]) as string | null,
failed: JSON.stringify(extra.failed ?? []) as string | null,
...("storageTotal" in extra ? { storageTotal: extra.storageTotal ?? null } : {}),
...("storageAvailable" in extra ? { storageAvailable: extra.storageAvailable ?? null } : {}),
});
const keys = (r: ReturnType<typeof row>) => pl.paperlessProblems(1, "Docs", r).map((p) => `${p.key}=${p.severity}`);
test("nothing wrong says nothing; what could not be read is not a problem", () => {
assert.deepEqual(keys(row()), []);
assert.deepEqual(pl.paperlessProblems(1, "Docs", { storageTotal: null, storageAvailable: null, components: null, failed: null }), []);
});
test("the database, Redis and the worker in error are critical; the index and the classifier a warning; an untrained classifier ('warning') says nothing", () => {
assert.deepEqual(keys(row({ components: [comp("celery", "ERROR", "No celery workers responded"), comp("index", "ERROR"), comp("classifier", "WARNING", "not trained")] })), ["paperless-component:1:celery=critical", "paperless-component:1:index=warning"]);
assert.match(pl.paperlessProblems(1, "Docs", row({ components: [comp("celery", "ERROR", "No celery workers responded")] }))[0].message, /the worker that consumes documents is in error — No celery workers responded/);
assert.deepEqual(keys(row({ components: [comp("database", "ERROR")] })), ["paperless-component:1:database=critical"]);
});
test("a sanity check that found problems is a warning", () => {
assert.deepEqual(keys(row({ components: [comp("sanity_check", "WARNING", "2 documents are missing their file")] })), ["paperless-component:1:sanity_check=warning"]);
});
test("imports that failed are one alert, naming the files and the latest error", () => {
const failed = ["a.pdf", "b.pdf", "c.pdf", "d.pdf"].map((file, i) => ({ file, error: i === 0 ? "Not a valid PDF" : null, at: null }));
const p = pl.paperlessProblems(1, "Docs", row({ failed }));
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["paperless-failed:1", "warning"]]);
assert.match(p[0].message, /4 documents failed to import \(a\.pdf, b\.pdf, c\.pdf and 1 more\) — the latest: Not a valid PDF/);
assert.match(pl.paperlessProblems(1, "Docs", row({ failed: [failed[1]] }))[0].message, /1 document failed to import \(b\.pdf\)$/);
});
test("the disk: a warning at 90 %, critical at 95 %", () => {
assert.deepEqual(keys(row({ storageAvailable: 1.1e11 })), []);
assert.deepEqual(keys(row({ storageAvailable: 1e11 })), ["paperless-disk:1=warning"]);
assert.deepEqual(keys(row({ storageAvailable: 4e10 })), ["paperless-disk:1=critical"]);
});
});
describe("the health pass, the page and /metrics", () => {
let plId = 0;
let app: Server;
let base = "";
const fresh = () => setInternalFlag("paperlessSync", JSON.stringify({ [plId]: { at: new Date().toISOString(), ok: true, error: null } }));
const stale = () => setInternalFlag("paperlessSync", JSON.stringify({ [plId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
plId = (await db.insert(integrations).values({ type: "paperless", name: "Archive", baseUrl: "http://10.9.9.0:8000", config: JSON.stringify({ url: "http://10.9.9.0:8000", token: "tok" }) }).returning({ id: integrations.id }))[0].id;
await pl.storeRead(
plId,
read({ storage: { total: 1e12, available: 3e10 }, components: [comp("database"), comp("celery", "ERROR", "No celery workers responded")], failed: [{ file: "scan.pdf", error: "Not a valid PDF", at: new Date().toISOString() }] }),
new Date(),
);
await fresh();
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/paperless", paperlessRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/paperless`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
let { snapshot, held } = await collectSnapshot();
assert.deepEqual(snapshot.paperless?.map((p) => p.integrationId), [plId]);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const c = list.find((x) => x.key === `paperless-component:${plId}:celery`)!;
assert.deepEqual([c.severity, c.source], ["critical", `paperless:${plId}`]);
assert.equal(subjectOfConditionKey(c.key), `integration:${plId}`, "maintenance on it silences its alerts");
for (const k of ["paperless-failed", "paperless-disk"]) assert.equal(subjectOfConditionKey(`${k}:${plId}`), `integration:${plId}`);
assert.ok(list.some((x) => x.key === `paperless-failed:${plId}`) && list.some((x) => x.key === `paperless-disk:${plId}`));
await stale();
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.paperless ?? [], held.has(`paperless:${plId}`)], [[], true]);
await fresh();
});
test("the Alerts page shows them: the worker and the failed imports under Services, the disk under Disk, with where to look", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const worker = report.alerts.find((x) => x.id === `services:paperless-component:${plId}:celery`)!;
assert.deepEqual([worker.severity, worker.category, worker.source, worker.link, worker.target?.type, worker.target?.id], ["critical", "services", "Paperless-ngx", "/paperless", "integration", plId]);
assert.equal(report.alerts.find((x) => x.id === `services:paperless-failed:${plId}`)?.category, "services");
assert.equal(report.alerts.find((x) => x.id === `disk:paperless-disk:${plId}`)?.severity, "critical");
});
test("anyone signed in sees each Paperless, its components and failed imports; an old reading says nothing is wrong", async () => {
const body = (await (await fetch(base)).json()) as { paperlesses: { id: number; fresh: boolean; state: { documents: number; components: { name: string; status: string }[]; failed: { file: string }[] } | null; problems: unknown[] }[] };
const p = body.paperlesses.find((x) => x.id === plId)!;
assert.equal(p.fresh, true);
assert.equal(p.state?.documents, 4200);
assert.deepEqual(p.state?.components.map((c) => [c.name, c.status]), [["database", "OK"], ["celery", "ERROR"]]);
assert.deepEqual(p.state?.failed.map((f) => f.file), ["scan.pdf"]);
assert.equal(p.problems.length, 3);
await stale();
const old = (await (await fetch(base)).json()) as typeof body;
assert.deepEqual(old.paperlesses.find((x) => x.id === plId)!.problems, []);
await fresh();
});
test("/metrics has the documents, the inbox, the failed imports and each component — only while the reading is recent", async () => {
let text = await buildMetrics(new Date());
assert.match(text, /homelab_paperless_documents\{name="Archive"\} 4200/);
assert.match(text, /homelab_paperless_inbox\{name="Archive"\} 3/);
assert.match(text, /homelab_paperless_failed_imports\{name="Archive"\} 1/);
assert.match(text, /homelab_paperless_component_ok\{name="Archive",component="celery"\} 0/);
assert.match(text, /homelab_paperless_component_ok\{name="Archive",component="database"\} 1/);
await stale();
text = await buildMetrics(new Date());
assert.doesNotMatch(text, /homelab_paperless/);
});
});