Three read-only integrations, each with a monitoring page, alerts, maintenance silencing and held-when-unreadable handling. Prometheus: scrape targets that stop answering (5 minutes, one alert per job) and alerts that Prometheus's own rules fire (critical when labelled so). Immich: disk under the library filling, failed background jobs, counts. Paperless-ngx: worker/database/Redis in error, documents that failed to import, index and sanity check, disk. Immich and Paperless also in /metrics. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
295 lines
17 KiB
TypeScript
295 lines
17 KiB
TypeScript
/**
|
|
* Paperless-ngx: reading what its API says (status components, statistics, failed imports), what is stored, what is wrong, the health pass, the alerts, the page's route and /metrics.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { createServer as createHttpServer, type Server } from "node:http";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { AddressInfo } from "node:net";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-paperless-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const express = (await import("express")).default;
|
|
const { db } = await import("../src/db/client.js");
|
|
const { users, integrations, paperlessState } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const adapter = await import("../src/integrations/paperless/adapter.js");
|
|
const pl = await import("../src/services/paperlessSync.js");
|
|
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
|
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
|
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
|
const { paperlessRouter } = await import("../src/routes/paperless.js");
|
|
const { buildMetrics } = await import("../src/services/metrics.js");
|
|
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
|
const { eq } = await import("drizzle-orm");
|
|
|
|
await runMigrations();
|
|
|
|
const MIN = 60_000;
|
|
const DAY = 86_400_000;
|
|
const NOW = new Date("2026-10-20T12:00:00Z");
|
|
|
|
type Read = import("../src/integrations/paperless/adapter.js").PaperlessRead;
|
|
const comp = (name: string, status = "OK", error: string | null = null) => ({ name, status, error });
|
|
const read = (extra: Partial<Read> = {}): Read => ({
|
|
version: "2.14.7", latestVersion: "2.14.7", updateAvailable: false, storage: { total: 1e12, available: 6e11 },
|
|
components: [comp("database"), comp("redis"), comp("celery"), comp("index"), comp("classifier"), comp("sanity_check")], documents: 4200, inbox: 3, failed: [], ...extra,
|
|
});
|
|
|
|
const STATUS = {
|
|
pngx_version: "2.14.7",
|
|
storage: { total: 1e12, available: 6e11 },
|
|
database: { type: "sqlite", status: "OK", error: null },
|
|
tasks: { redis_status: "OK", celery_status: "ERROR", celery_error: "No celery workers responded", index_status: "OK", classifier_status: "WARNING", classifier_error: "Classifier has not been trained yet", sanity_check_status: "OK" },
|
|
};
|
|
|
|
describe("what Paperless's API says", () => {
|
|
test("the system status: the version, the disk and each component with how it is", () => {
|
|
const s = adapter.parseStatus(STATUS);
|
|
assert.equal(s.version, "2.14.7");
|
|
assert.deepEqual(s.storage, { total: 1e12, available: 6e11 });
|
|
assert.deepEqual(s.components?.map((c) => [c.name, c.status, c.error]), [["database", "OK", null], ["redis", "OK", null], ["celery", "ERROR", "No celery workers responded"], ["index", "OK", null], ["classifier", "WARNING", "Classifier has not been trained yet"], ["sanity_check", "OK", null]]);
|
|
assert.deepEqual(adapter.parseStatus({}), { version: null, storage: null, components: null });
|
|
});
|
|
|
|
test("the documents and the inbox", () => {
|
|
assert.deepEqual(adapter.parseStatistics({ documents_total: 4200, documents_inbox: 3 }), { documents: 4200, inbox: 3 });
|
|
assert.deepEqual(adapter.parseStatistics(null), { documents: null, inbox: null });
|
|
});
|
|
|
|
test("failed imports: only failures, not dismissed, from the last week, newest first; an answer without a list is not 'none'", () => {
|
|
const f = adapter.parseFailures(
|
|
[
|
|
{ status: "FAILURE", task_file_name: "old.pdf", date_done: new Date(NOW.getTime() - 10 * DAY).toISOString(), result: "x" },
|
|
{ status: "FAILURE", task_file_name: "a.pdf", date_done: new Date(NOW.getTime() - 2 * DAY).toISOString(), result: "Not a valid PDF" },
|
|
{ status: "FAILURE", task_file_name: "dismissed.pdf", acknowledged: true, date_done: NOW.toISOString() },
|
|
{ status: "SUCCESS", task_file_name: "fine.pdf", date_done: NOW.toISOString() },
|
|
{ status: "FAILURE", task_file_name: "b.pdf", date_done: new Date(NOW.getTime() - 3600_000).toISOString() },
|
|
],
|
|
NOW,
|
|
)!;
|
|
assert.deepEqual(f.map((x) => [x.file, x.error]), [["b.pdf", null], ["a.pdf", "Not a valid PDF"]]);
|
|
assert.equal(adapter.parseFailures({ results: [] }, NOW)?.length, 0, "a paginated answer is read too");
|
|
assert.equal(adapter.parseFailures({ detail: "no" }, NOW), null);
|
|
});
|
|
|
|
test("a newer version", () => {
|
|
assert.deepEqual(adapter.parseRemoteVersion({ version: "v2.15.0", update_available: true }), { latestVersion: "v2.15.0", updateAvailable: true });
|
|
assert.deepEqual(adapter.parseRemoteVersion(null), { latestVersion: null, updateAvailable: false });
|
|
});
|
|
|
|
describe("the adapter against a Paperless that answers", () => {
|
|
let server: Server;
|
|
let url = "";
|
|
let superuser = true;
|
|
const seen: (string | undefined)[] = [];
|
|
before(async () => {
|
|
server = createHttpServer((req, res) => {
|
|
seen.push(req.headers.authorization);
|
|
const send = (code: number, body: unknown) => {
|
|
res.writeHead(code, { "content-type": "application/json" });
|
|
res.end(JSON.stringify(body));
|
|
};
|
|
if (req.headers.authorization !== "Token tok") return send(401, {});
|
|
const path = (req.url ?? "").split("?")[0];
|
|
if (path === "/api/statistics/") return send(200, { documents_total: 100, documents_inbox: 2 });
|
|
if (path === "/api/status/") return superuser ? send(200, STATUS) : send(403, {});
|
|
if (path === "/api/tasks/") return send(200, [{ status: "FAILURE", task_file_name: "x.pdf", date_done: new Date().toISOString(), result: "boom" }]);
|
|
if (path === "/api/remote_version/") return send(200, { version: "2.15.0", update_available: true });
|
|
send(404, {});
|
|
});
|
|
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
|
});
|
|
after(() => server.close());
|
|
|
|
test("it reads everything and sends the token", async () => {
|
|
const r = await adapter.createPaperlessAdapter({ url: `${url}/`, token: "tok" }).read(new Date());
|
|
assert.deepEqual([r.version, r.documents, r.inbox, r.updateAvailable, r.latestVersion], ["2.14.7", 100, 2, true, "2.15.0"]);
|
|
assert.deepEqual(r.failed?.map((f) => f.file), ["x.pdf"]);
|
|
assert.equal(r.components?.find((c) => c.name === "celery")?.status, "ERROR");
|
|
assert.ok(seen.every((s) => s === "Token tok"));
|
|
});
|
|
|
|
test("a token that is not a superuser's still reads the rest; the status is unknown, not fine", async () => {
|
|
superuser = false;
|
|
const r = await adapter.createPaperlessAdapter({ url, token: "tok" }).read(new Date());
|
|
assert.deepEqual([r.components, r.storage, r.version], [null, null, null]);
|
|
assert.equal(r.documents, 100);
|
|
});
|
|
|
|
test("a refused token and a dead address are said plainly", async () => {
|
|
const bad = await adapter.createPaperlessAdapter({ url, token: "nope" }).ping();
|
|
assert.equal(bad.ok, false);
|
|
assert.match(bad.error ?? "", /did not accept the token/);
|
|
assert.equal((await adapter.createPaperlessAdapter({ url, token: "tok" }).ping()).ok, true);
|
|
assert.equal((await adapter.createPaperlessAdapter({ url: "http://127.0.0.1:1", token: "tok" }).ping()).ok, false);
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("what is stored", () => {
|
|
let id = 0;
|
|
before(async () => {
|
|
id = (await db.insert(integrations).values({ type: "paperless", name: "Documents", baseUrl: "http://10.9.9.1:8000", config: JSON.stringify({ url: "http://10.9.9.1:8000", token: "tok" }) }).returning({ id: integrations.id }))[0].id;
|
|
});
|
|
const state = async () => (await db.select().from(paperlessState).where(eq(paperlessState.integrationId, id)))[0];
|
|
|
|
test("the state is replaced at each read; what could not be read stays null, not empty", async () => {
|
|
await pl.storeRead(id, read({ failed: [{ file: "a.pdf", error: null, at: null }] }), NOW);
|
|
assert.deepEqual(pl.failedOf(await state())?.map((f) => f.file), ["a.pdf"]);
|
|
await pl.storeRead(id, read({ components: null, failed: null, storage: null }), NOW);
|
|
const s = await state();
|
|
assert.deepEqual([pl.componentsOf(s), pl.failedOf(s), s.storageTotal], [null, null, null]);
|
|
});
|
|
|
|
test("a read that failed is said, and what was stored is left alone", async () => {
|
|
const ok = await pl.syncPaperless(id, new Date(), async () => read({ version: "2.15.0" }));
|
|
assert.equal(ok.ok, true);
|
|
const failed = await pl.syncPaperless(id, new Date(), async () => {
|
|
throw new Error("connection refused");
|
|
});
|
|
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
|
|
assert.equal(failed.at, ok.at);
|
|
assert.equal((await state()).version, "2.15.0");
|
|
assert.equal(pl.isPaperlessFresh((await pl.paperlessSyncStatuses())[id], Date.now()), false);
|
|
});
|
|
});
|
|
|
|
describe("what is wrong", () => {
|
|
const row = (extra: { storageTotal?: number | null; storageAvailable?: number | null; components?: unknown; failed?: unknown } = {}) => ({
|
|
storageTotal: 1e12 as number | null,
|
|
storageAvailable: 6e11 as number | null,
|
|
components: JSON.stringify(extra.components ?? [comp("database"), comp("celery")]) as string | null,
|
|
failed: JSON.stringify(extra.failed ?? []) as string | null,
|
|
...("storageTotal" in extra ? { storageTotal: extra.storageTotal ?? null } : {}),
|
|
...("storageAvailable" in extra ? { storageAvailable: extra.storageAvailable ?? null } : {}),
|
|
});
|
|
const keys = (r: ReturnType<typeof row>) => pl.paperlessProblems(1, "Docs", r).map((p) => `${p.key}=${p.severity}`);
|
|
|
|
test("nothing wrong says nothing; what could not be read is not a problem", () => {
|
|
assert.deepEqual(keys(row()), []);
|
|
assert.deepEqual(pl.paperlessProblems(1, "Docs", { storageTotal: null, storageAvailable: null, components: null, failed: null }), []);
|
|
});
|
|
|
|
test("the database, Redis and the worker in error are critical; the index and the classifier a warning; an untrained classifier ('warning') says nothing", () => {
|
|
assert.deepEqual(keys(row({ components: [comp("celery", "ERROR", "No celery workers responded"), comp("index", "ERROR"), comp("classifier", "WARNING", "not trained")] })), ["paperless-component:1:celery=critical", "paperless-component:1:index=warning"]);
|
|
assert.match(pl.paperlessProblems(1, "Docs", row({ components: [comp("celery", "ERROR", "No celery workers responded")] }))[0].message, /the worker that consumes documents is in error — No celery workers responded/);
|
|
assert.deepEqual(keys(row({ components: [comp("database", "ERROR")] })), ["paperless-component:1:database=critical"]);
|
|
});
|
|
|
|
test("a sanity check that found problems is a warning", () => {
|
|
assert.deepEqual(keys(row({ components: [comp("sanity_check", "WARNING", "2 documents are missing their file")] })), ["paperless-component:1:sanity_check=warning"]);
|
|
});
|
|
|
|
test("imports that failed are one alert, naming the files and the latest error", () => {
|
|
const failed = ["a.pdf", "b.pdf", "c.pdf", "d.pdf"].map((file, i) => ({ file, error: i === 0 ? "Not a valid PDF" : null, at: null }));
|
|
const p = pl.paperlessProblems(1, "Docs", row({ failed }));
|
|
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["paperless-failed:1", "warning"]]);
|
|
assert.match(p[0].message, /4 documents failed to import \(a\.pdf, b\.pdf, c\.pdf and 1 more\) — the latest: Not a valid PDF/);
|
|
assert.match(pl.paperlessProblems(1, "Docs", row({ failed: [failed[1]] }))[0].message, /1 document failed to import \(b\.pdf\)$/);
|
|
});
|
|
|
|
test("the disk: a warning at 90 %, critical at 95 %", () => {
|
|
assert.deepEqual(keys(row({ storageAvailable: 1.1e11 })), []);
|
|
assert.deepEqual(keys(row({ storageAvailable: 1e11 })), ["paperless-disk:1=warning"]);
|
|
assert.deepEqual(keys(row({ storageAvailable: 4e10 })), ["paperless-disk:1=critical"]);
|
|
});
|
|
});
|
|
|
|
describe("the health pass, the page and /metrics", () => {
|
|
let plId = 0;
|
|
let app: Server;
|
|
let base = "";
|
|
const fresh = () => setInternalFlag("paperlessSync", JSON.stringify({ [plId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
|
const stale = () => setInternalFlag("paperlessSync", JSON.stringify({ [plId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
|
|
|
|
before(async () => {
|
|
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
|
plId = (await db.insert(integrations).values({ type: "paperless", name: "Archive", baseUrl: "http://10.9.9.0:8000", config: JSON.stringify({ url: "http://10.9.9.0:8000", token: "tok" }) }).returning({ id: integrations.id }))[0].id;
|
|
await pl.storeRead(
|
|
plId,
|
|
read({ storage: { total: 1e12, available: 3e10 }, components: [comp("database"), comp("celery", "ERROR", "No celery workers responded")], failed: [{ file: "scan.pdf", error: "Not a valid PDF", at: new Date().toISOString() }] }),
|
|
new Date(),
|
|
);
|
|
await fresh();
|
|
|
|
const a = express();
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
|
next();
|
|
});
|
|
a.use("/api/paperless", paperlessRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/paperless`;
|
|
});
|
|
after(() => {
|
|
app.close();
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows keeps the database file locked until the process exits.
|
|
}
|
|
});
|
|
|
|
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
|
|
let { snapshot, held } = await collectSnapshot();
|
|
assert.deepEqual(snapshot.paperless?.map((p) => p.integrationId), [plId]);
|
|
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
|
const c = list.find((x) => x.key === `paperless-component:${plId}:celery`)!;
|
|
assert.deepEqual([c.severity, c.source], ["critical", `paperless:${plId}`]);
|
|
assert.equal(subjectOfConditionKey(c.key), `integration:${plId}`, "maintenance on it silences its alerts");
|
|
for (const k of ["paperless-failed", "paperless-disk"]) assert.equal(subjectOfConditionKey(`${k}:${plId}`), `integration:${plId}`);
|
|
assert.ok(list.some((x) => x.key === `paperless-failed:${plId}`) && list.some((x) => x.key === `paperless-disk:${plId}`));
|
|
await stale();
|
|
({ snapshot, held } = await collectSnapshot());
|
|
assert.deepEqual([snapshot.paperless ?? [], held.has(`paperless:${plId}`)], [[], true]);
|
|
await fresh();
|
|
});
|
|
|
|
test("the Alerts page shows them: the worker and the failed imports under Services, the disk under Disk, with where to look", async () => {
|
|
invalidateAlertsCache();
|
|
const report = await getAlerts(true);
|
|
const worker = report.alerts.find((x) => x.id === `services:paperless-component:${plId}:celery`)!;
|
|
assert.deepEqual([worker.severity, worker.category, worker.source, worker.link, worker.target?.type, worker.target?.id], ["critical", "services", "Paperless-ngx", "/paperless", "integration", plId]);
|
|
assert.equal(report.alerts.find((x) => x.id === `services:paperless-failed:${plId}`)?.category, "services");
|
|
assert.equal(report.alerts.find((x) => x.id === `disk:paperless-disk:${plId}`)?.severity, "critical");
|
|
});
|
|
|
|
test("anyone signed in sees each Paperless, its components and failed imports; an old reading says nothing is wrong", async () => {
|
|
const body = (await (await fetch(base)).json()) as { paperlesses: { id: number; fresh: boolean; state: { documents: number; components: { name: string; status: string }[]; failed: { file: string }[] } | null; problems: unknown[] }[] };
|
|
const p = body.paperlesses.find((x) => x.id === plId)!;
|
|
assert.equal(p.fresh, true);
|
|
assert.equal(p.state?.documents, 4200);
|
|
assert.deepEqual(p.state?.components.map((c) => [c.name, c.status]), [["database", "OK"], ["celery", "ERROR"]]);
|
|
assert.deepEqual(p.state?.failed.map((f) => f.file), ["scan.pdf"]);
|
|
assert.equal(p.problems.length, 3);
|
|
await stale();
|
|
const old = (await (await fetch(base)).json()) as typeof body;
|
|
assert.deepEqual(old.paperlesses.find((x) => x.id === plId)!.problems, []);
|
|
await fresh();
|
|
});
|
|
|
|
test("/metrics has the documents, the inbox, the failed imports and each component — only while the reading is recent", async () => {
|
|
let text = await buildMetrics(new Date());
|
|
assert.match(text, /homelab_paperless_documents\{name="Archive"\} 4200/);
|
|
assert.match(text, /homelab_paperless_inbox\{name="Archive"\} 3/);
|
|
assert.match(text, /homelab_paperless_failed_imports\{name="Archive"\} 1/);
|
|
assert.match(text, /homelab_paperless_component_ok\{name="Archive",component="celery"\} 0/);
|
|
assert.match(text, /homelab_paperless_component_ok\{name="Archive",component="database"\} 1/);
|
|
await stale();
|
|
text = await buildMetrics(new Date());
|
|
assert.doesNotMatch(text, /homelab_paperless/);
|
|
});
|
|
});
|