Files
Homelab-manager/server/src/services/paperlessSync.ts
T
bobbanandClaude Sonnet 5.5 251a4a9d73 Prometheus, Immich and Paperless-ngx integrations
Three read-only integrations, each with a monitoring page, alerts, maintenance
silencing and held-when-unreadable handling.

Prometheus: scrape targets that stop answering (5 minutes, one alert per job)
and alerts that Prometheus's own rules fire (critical when labelled so).
Immich: disk under the library filling, failed background jobs, counts.
Paperless-ngx: worker/database/Redis in error, documents that failed to
import, index and sanity check, disk. Immich and Paperless also in /metrics.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-24 22:33:17 +02:00

163 lines
8.3 KiB
TypeScript

/**
* Keeps what Paperless-ngx says up to date: every five minutes each Paperless integration is read (its system status, the disk, the document counts and the imports that failed)
* and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Paperless.
*
* Paperless fails quietly in two ways: the worker that consumes documents stops (a scan dropped in the folder just sits there), or one import fails and nobody looks. Both are raised,
* and so is anything Paperless's own status check says is in error.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, paperlessState } from "../db/schema.js";
import { createPaperlessAdapter, type PaperlessComponent, type PaperlessFailure, type PaperlessRead } from "../integrations/paperless/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 5 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 20 * 60_000;
export const DISK_WARN_PERCENT = 90;
export const DISK_CRITICAL_PERCENT = 95;
/** If one of these is in error, nothing can be consumed or found: critical. The others are a warning. */
const ESSENTIAL = new Set(["database", "redis", "celery"]);
const STATUS_FLAG = "paperlessSync";
export interface PaperlessSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, PaperlessSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, PaperlessSyncStatus>) : {};
} catch {
return {};
}
}
export async function paperlessSyncStatuses(): Promise<Record<number, PaperlessSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isPaperlessFresh = (status: PaperlessSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof paperlessState.$inferSelect;
/** Null stays null: a list that could not be read is not an empty one. */
const parse = <T>(text: string | null): T[] | null => {
if (text === null) return null;
try {
const v = JSON.parse(text);
return Array.isArray(v) ? (v as T[]) : null;
} catch {
return null;
}
};
export const componentsOf = (row: Pick<StateRow, "components">) => parse<PaperlessComponent>(row.components);
export const failedOf = (row: Pick<StateRow, "failed">) => parse<PaperlessFailure>(row.failed);
/** Writes down one read, replacing the last. */
export async function storeRead(integrationId: number, r: PaperlessRead, now: Date = new Date()): Promise<void> {
const values = {
version: r.version,
latestVersion: r.latestVersion,
updateAvailable: r.updateAvailable,
storageTotal: r.storage?.total ?? null,
storageAvailable: r.storage?.available ?? null,
components: r.components && JSON.stringify(r.components),
documents: r.documents,
inbox: r.inbox,
failed: r.failed && JSON.stringify(r.failed),
updatedAt: now.toISOString(),
};
await db.insert(paperlessState).values({ integrationId, ...values }).onConflictDoUpdate({ target: paperlessState.integrationId, set: values });
}
/** Reads one Paperless and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncPaperless(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<PaperlessRead>): Promise<PaperlessSyncStatus> {
let status: PaperlessSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "paperless") throw new Error("not a Paperless integration");
const result = await (read ? read(loaded.config) : createPaperlessAdapter(loaded.config as never).read(now));
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[paperless] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[paperless] couldn't record the result:", err);
}
return status;
}
export async function syncAllPaperless(now: Date = new Date()): Promise<Record<number, PaperlessSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "paperless"), eq(integrations.enabled, true)));
const out: Record<number, PaperlessSyncStatus> = {};
for (const r of rows) out[r.id] = await syncPaperless(r.id, now);
return out;
}
export async function forgetPaperless(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface PaperlessProblem {
key: string;
message: string;
severity: "warning" | "critical";
}
const gb = (bytes: number) => `${(bytes / 1e9).toFixed(bytes >= 1e11 ? 0 : 1)} GB`;
const WORDS: Record<string, string> = { database: "the database", redis: "Redis", celery: "the worker that consumes documents", index: "the search index", classifier: "the classifier", sanity_check: "the sanity check" };
/** Pure: what is wrong with one Paperless, from what is stored for it. */
export function paperlessProblems(id: number, name: string, s: Pick<StateRow, "storageTotal" | "storageAvailable" | "components" | "failed">): PaperlessProblem[] {
const out: PaperlessProblem[] = [];
for (const c of componentsOf(s) ?? []) {
const word = WORDS[c.name] ?? c.name;
if (c.status === "ERROR") {
out.push({ key: `paperless-component:${id}:${c.name}`, severity: ESSENTIAL.has(c.name) ? "critical" : "warning", message: `${name}: ${word} is in error${c.error ? ` — ${c.error.length > 140 ? `${c.error.slice(0, 137)}…` : c.error}` : ""}` });
} else if (c.status === "WARNING" && c.name === "sanity_check") {
out.push({ key: `paperless-component:${id}:${c.name}`, severity: "warning", message: `${name}: the sanity check found problems with some documents${c.error ? ` — ${c.error.length > 140 ? `${c.error.slice(0, 137)}…` : c.error}` : ""}` });
}
}
const failed = failedOf(s) ?? [];
if (failed.length > 0) {
const files = failed.slice(0, 3).map((f) => f.file).join(", ") + (failed.length > 3 ? ` and ${failed.length - 3} more` : "");
const why = failed[0].error ? ` — the latest: ${failed[0].error.length > 140 ? `${failed[0].error.slice(0, 137)}…` : failed[0].error}` : "";
out.push({ key: `paperless-failed:${id}`, severity: "warning", message: `${name}: ${failed.length} document${failed.length === 1 ? "" : "s"} failed to import (${files})${why}` });
}
if (s.storageTotal !== null && s.storageAvailable !== null && s.storageTotal > 0) {
const percent = ((s.storageTotal - s.storageAvailable) / s.storageTotal) * 100;
if (percent >= DISK_WARN_PERCENT) {
out.push({ key: `paperless-disk:${id}`, severity: percent >= DISK_CRITICAL_PERCENT ? "critical" : "warning", message: `${name}: the disk under the documents is ${Math.round(percent)} % full (${gb(s.storageTotal - s.storageAvailable)} of ${gb(s.storageTotal)})` });
}
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initPaperlessSync(): void {
if (timer) return;
const pass = () => void syncAllPaperless().catch((err) => console.error("[paperless] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 19_000);
console.log(`[paperless] reading Paperless every ${SYNC_INTERVAL_MS / 60000} minutes`);
}