Prometheus, Immich and Paperless-ngx integrations

Three read-only integrations, each with a monitoring page, alerts, maintenance
silencing and held-when-unreadable handling.

Prometheus: scrape targets that stop answering (5 minutes, one alert per job)
and alerts that Prometheus's own rules fire (critical when labelled so).
Immich: disk under the library filling, failed background jobs, counts.
Paperless-ngx: worker/database/Redis in error, documents that failed to
import, index and sanity check, disk. Immich and Paperless also in /metrics.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-24 22:33:17 +02:00
1 parent aebe98bf15
commit 251a4a9d73
47 files changed
+24148 -7

No files matched your search

@@ -0,0 +1,140 @@
/**
* Prometheus adapter. Config: url; optional: token (a bearer token, when something in front of Prometheus asks for one), insecure.
*
* Reading only: Prometheus's version, every scrape target and whether its last scrape worked (a node exporter that stopped answering), and the alerts that Prometheus's own rules have
* pending or firing. That makes Prometheus a source: whatever you already measure there is seen here too. Nothing is sent to Prometheus but GET requests for these.
*
* Endpoints and field names are from Prometheus's published HTTP API (`/api/v1/status/buildinfo`, `/api/v1/targets`, `/api/v1/alerts`) and have not been checked against a live server
* here; every field is read defensively. Alerts that Grafana manages itself are not in Prometheus's list and are not seen.
*/
import { requestJson } from "../httpJson.js";
import { withDiagLogging } from "../../services/diagLog.js";
export interface PrometheusConfig {
url: string;
token?: string;
insecure?: boolean;
}
export interface PrometheusTarget {
job: string;
instance: string;
health: "up" | "down" | "unknown";
/** Why the last scrape failed. */
error: string | null;
lastScrape: string | null;
}
export interface PrometheusAlert {
name: string;
/** From the alert's own `severity` label, lower-case; null when it has none. */
severity: string | null;
state: "firing" | "pending";
summary: string | null;
instance: string | null;
activeAt: string | null;
}
export interface PrometheusRead {
version: string | null;
/** Null when the targets could not be asked, which is not the same as there being none. */
targets: PrometheusTarget[] | null;
alerts: PrometheusAlert[] | null;
}
export interface PrometheusAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
read(): Promise<PrometheusRead>;
}
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
const iso = (v: unknown): string | null => {
const s = str(v);
return s && Number.isFinite(Date.parse(s)) ? new Date(s).toISOString() : null;
};
/** Pure: the version from /status/buildinfo. */
export function parseBuildInfo(raw: unknown): string | null {
return str(obj(obj(raw).data).version);
}
/** Pure: the active targets from /targets; null when the answer has no list in it. */
export function parseTargets(raw: unknown): PrometheusTarget[] | null {
const list = obj(obj(raw).data).activeTargets;
if (!Array.isArray(list)) return null;
const out: PrometheusTarget[] = [];
for (const item of list) {
const t = obj(item);
const labels = obj(t.labels);
const job = str(labels.job) ?? str(t.scrapePool);
if (!job) continue;
const health = str(t.health)?.toLowerCase();
out.push({
job,
instance: str(labels.instance) ?? str(t.scrapeUrl) ?? "?",
health: health === "up" || health === "down" ? health : "unknown",
error: str(t.lastError),
lastScrape: iso(t.lastScrape),
});
}
return out;
}
/** Pure: the alerts from /alerts; null when the answer has no list in it. */
export function parseAlerts(raw: unknown): PrometheusAlert[] | null {
const list = obj(obj(raw).data).alerts;
if (!Array.isArray(list)) return null;
const out: PrometheusAlert[] = [];
for (const item of list) {
const a = obj(item);
const labels = obj(a.labels);
const annotations = obj(a.annotations);
const name = str(labels.alertname);
const state = str(a.state)?.toLowerCase();
if (!name || (state !== "firing" && state !== "pending")) continue;
out.push({ name, severity: str(labels.severity)?.toLowerCase() ?? null, state, summary: str(annotations.summary) ?? str(annotations.description) ?? str(annotations.message), instance: str(labels.instance), activeAt: iso(a.activeAt) });
}
return out;
}
export function createPrometheusAdapter(config: PrometheusConfig): PrometheusAdapter {
const base = String(config.url ?? "").replace(/\/+$/, "");
const insecure = config.insecure === true;
const headers: Record<string, string> = config.token ? { Authorization: `Bearer ${config.token}` } : {};
async function get(path: string): Promise<unknown> {
const res = await requestJson(`${base}${path}`, { headers, insecure });
if (res.status === 401 || res.status === 403) throw new Error(`Prometheus did not accept the request (HTTP ${res.status}) — something in front of it wants a token`);
if (res.status === 404) throw new Error("Prometheus has no API at this address — check the address and port (9090)");
if (res.status < 200 || res.status >= 300) throw new Error(`Prometheus error: HTTP ${res.status}`);
if (res.json === null) throw new Error("Prometheus answered with something that isn't JSON — is the address its own?");
return res.json;
}
async function read(): Promise<PrometheusRead> {
const version = parseBuildInfo(await get("/api/v1/status/buildinfo"));
// The rest is worth having but not worth failing for.
const soft = async <T>(path: string, parse: (raw: unknown) => T): Promise<T | null> => {
try {
return parse(await get(path));
} catch {
return null;
}
};
const [targets, alerts] = await Promise.all([soft("/api/v1/targets?state=active", parseTargets), soft("/api/v1/alerts", parseAlerts)]);
return { version, targets, alerts };
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
const start = Date.now();
try {
await get("/api/v1/status/buildinfo");
return { ok: true, latencyMs: Date.now() - start };
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) };
}
}
return withDiagLogging("prometheus", { ping, read });
}