Prometheus, Immich and Paperless-ngx integrations
Three read-only integrations, each with a monitoring page, alerts, maintenance silencing and held-when-unreadable handling. Prometheus: scrape targets that stop answering (5 minutes, one alert per job) and alerts that Prometheus's own rules fire (critical when labelled so). Immich: disk under the library filling, failed background jobs, counts. Paperless-ngx: worker/database/Redis in error, documents that failed to import, index and sanity check, disk. Immich and Paperless also in /metrics. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aebe98bf15
commit
251a4a9d73
47 files changed
+24148
-7
No files matched your search
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* Keeps what Immich says up to date: every five minutes each Immich integration is read (version, the disk under the library, the photo and video counts, the background job
|
||||
* queues) and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Immich.
|
||||
*
|
||||
* Photos are what cannot be made again, and Immich fails quietly: a disk that fills up, or thumbnail and metadata jobs that fail one by one and are never looked at. Those are
|
||||
* what is raised.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { immichState, integrations } from "../db/schema.js";
|
||||
import { createImmichAdapter, type ImmichQueue, type ImmichRead } from "../integrations/immich/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
/** This many failed background jobs, together, is a warning. */
|
||||
export const FAILED_JOBS_WARN = 10;
|
||||
export const DISK_WARN_PERCENT = 90;
|
||||
export const DISK_CRITICAL_PERCENT = 95;
|
||||
const STATUS_FLAG = "immichSync";
|
||||
|
||||
export interface ImmichSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, ImmichSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, ImmichSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function immichSyncStatuses(): Promise<Record<number, ImmichSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isImmichFresh = (status: ImmichSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof immichState.$inferSelect;
|
||||
|
||||
/** Null stays null: a list that could not be read is not an empty one. */
|
||||
export const queuesOf = (row: Pick<StateRow, "queues">): ImmichQueue[] | null => {
|
||||
if (row.queues === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(row.queues);
|
||||
return Array.isArray(v) ? (v as ImmichQueue[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/** Writes down one read, replacing the last. */
|
||||
export async function storeRead(integrationId: number, r: ImmichRead, now: Date = new Date()): Promise<void> {
|
||||
const values = {
|
||||
version: r.version,
|
||||
latestVersion: r.latestVersion,
|
||||
updateAvailable: r.updateAvailable,
|
||||
diskSize: r.disk?.size ?? null,
|
||||
diskUsed: r.disk?.used ?? null,
|
||||
diskPercent: r.disk?.percent ?? null,
|
||||
photos: r.photos,
|
||||
videos: r.videos,
|
||||
usageBytes: r.usageBytes,
|
||||
queues: r.queues && JSON.stringify(r.queues),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(immichState).values({ integrationId, ...values }).onConflictDoUpdate({ target: immichState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one Immich and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncImmich(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<ImmichRead>): Promise<ImmichSyncStatus> {
|
||||
let status: ImmichSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "immich") throw new Error("not an Immich integration");
|
||||
const result = await (read ? read(loaded.config) : createImmichAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[immich] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[immich] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllImmich(now: Date = new Date()): Promise<Record<number, ImmichSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "immich"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, ImmichSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncImmich(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetImmich(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface ImmichProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const gb = (bytes: number) => `${(bytes / 1e9).toFixed(bytes >= 1e11 ? 0 : 1)} GB`;
|
||||
|
||||
/** Pure: what is wrong with one Immich, from what is stored for it. */
|
||||
export function immichProblems(id: number, name: string, s: Pick<StateRow, "diskSize" | "diskUsed" | "diskPercent" | "queues">): ImmichProblem[] {
|
||||
const out: ImmichProblem[] = [];
|
||||
if (s.diskPercent !== null && s.diskPercent >= DISK_WARN_PERCENT) {
|
||||
out.push({
|
||||
key: `immich-disk:${id}`,
|
||||
severity: s.diskPercent >= DISK_CRITICAL_PERCENT ? "critical" : "warning",
|
||||
message: `${name}: the disk under the library is ${Math.round(s.diskPercent)} % full${s.diskSize !== null && s.diskUsed !== null ? ` (${gb(s.diskUsed)} of ${gb(s.diskSize)})` : ""}`,
|
||||
});
|
||||
}
|
||||
const queues = queuesOf(s) ?? [];
|
||||
const failed = queues.filter((q) => q.failed > 0);
|
||||
const total = failed.reduce((n, q) => n + q.failed, 0);
|
||||
if (total >= FAILED_JOBS_WARN) {
|
||||
const worst = [...failed].sort((a, b) => b.failed - a.failed).slice(0, 3).map((q) => `${q.name} ${q.failed}`).join(", ");
|
||||
out.push({ key: `immich-jobs:${id}`, severity: "warning", message: `${name}: ${total} background jobs have failed (${worst}) — thumbnails or metadata that were never made; look at Administration → Jobs and run the failed ones again` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initImmichSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllImmich().catch((err) => console.error("[immich] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 17_000);
|
||||
console.log(`[immich] reading Immich every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
Reference in new issue
Block a user