Prometheus, Immich and Paperless-ngx integrations

Three read-only integrations, each with a monitoring page, alerts, maintenance
silencing and held-when-unreadable handling.

Prometheus: scrape targets that stop answering (5 minutes, one alert per job)
and alerts that Prometheus's own rules fire (critical when labelled so).
Immich: disk under the library filling, failed background jobs, counts.
Paperless-ngx: worker/database/Redis in error, documents that failed to
import, index and sanity check, disk. Immich and Paperless also in /metrics.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-24 22:33:17 +02:00
1 parent aebe98bf15
commit 251a4a9d73
47 files changed
+24148 -7

No files matched your search

+38 -1
View File
@@ -5,11 +5,13 @@
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState } from "../db/schema.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { backupState, listAll as listDuplicati } from "./duplicati.js";
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
import { componentsOf as paperlessComponentsOf, failedOf as paperlessFailedOf, isPaperlessFresh, paperlessSyncStatuses } from "./paperlessSync.js";
import { immichSyncStatuses, isImmichFresh, queuesOf as immichQueuesOf } from "./immichSync.js";
import { authentikSyncStatuses, certificatesOf, daysUntil, isAuthentikFresh, outpostsOf } from "./authentikSync.js";
import { latestTests } from "./restoreTests.js";
import { heartbeatState, parseStamp } from "./heartbeats.js";
@@ -62,6 +64,39 @@ class Exposition {
}
}
/** Each Paperless-ngx, for /metrics: the documents, the inbox, the imports that failed and whether each component of its status is in order. Only while the reading is recent. */
async function addPaperlessMetrics(out: Exposition, now: Date) {
const statuses = await paperlessSyncStatuses();
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "paperless"))).map((r) => [r.id, r]));
for (const s of await db.select().from(paperlessState)) {
const row = names.get(s.integrationId);
if (!row || !row.enabled || !isPaperlessFresh(statuses[s.integrationId], now.getTime())) continue;
const labels = { name: row.name };
if (s.documents !== null) out.add("homelab_paperless_documents", "gauge", "How many documents Paperless holds.", s.documents, labels);
if (s.inbox !== null) out.add("homelab_paperless_inbox", "gauge", "How many documents are waiting in Paperless's inbox.", s.inbox, labels);
const failed = paperlessFailedOf(s);
if (failed) out.add("homelab_paperless_failed_imports", "gauge", "Imports that failed in the last week and were not dismissed.", failed.length, labels);
for (const c of paperlessComponentsOf(s) ?? []) out.add("homelab_paperless_component_ok", "gauge", "1 if the part of Paperless named is in order, else 0.", c.status === "OK" ? 1 : 0, { name: row.name, component: c.name });
}
}
/** Each Immich, for /metrics: the photos and videos, what they take, how full the disk is and the jobs that failed. Only while the reading is recent. */
async function addImmichMetrics(out: Exposition, now: Date) {
const statuses = await immichSyncStatuses();
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "immich"))).map((r) => [r.id, r]));
for (const s of await db.select().from(immichState)) {
const row = names.get(s.integrationId);
if (!row || !row.enabled || !isImmichFresh(statuses[s.integrationId], now.getTime())) continue;
const labels = { name: row.name };
if (s.photos !== null) out.add("homelab_immich_photos", "gauge", "How many photos Immich holds.", s.photos, labels);
if (s.videos !== null) out.add("homelab_immich_videos", "gauge", "How many videos Immich holds.", s.videos, labels);
if (s.usageBytes !== null) out.add("homelab_immich_library_bytes", "gauge", "What Immich's library takes, in bytes.", s.usageBytes, labels);
if (s.diskPercent !== null) out.add("homelab_immich_disk_used_percent", "gauge", "How full the disk under Immich's library is.", s.diskPercent, labels);
const queues = immichQueuesOf(s);
if (queues) out.add("homelab_immich_failed_jobs", "gauge", "Background jobs that have failed, all queues together.", queues.reduce((n, q) => n + q.failed, 0), labels);
}
}
/** Each Authentik, for /metrics: whether each outpost that serves something is reporting, how many workers there are, the failed sign-ins and the days left of each certificate. Only while the reading is recent. */
async function addAuthentikMetrics(out: Exposition, now: Date) {
const statuses = await authentikSyncStatuses();
@@ -300,6 +335,8 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addDuplicatiMetrics(out, now);
await addJellyfinMetrics(out, now);
await addAuthentikMetrics(out, now);
await addImmichMetrics(out, now);
await addPaperlessMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}