Prometheus, Immich and Paperless-ngx integrations
Three read-only integrations, each with a monitoring page, alerts, maintenance silencing and held-when-unreadable handling. Prometheus: scrape targets that stop answering (5 minutes, one alert per job) and alerts that Prometheus's own rules fire (critical when labelled so). Immich: disk under the library filling, failed background jobs, counts. Paperless-ngx: worker/database/Redis in error, documents that failed to import, index and sanity check, disk. Immich and Paperless also in /metrics. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aebe98bf15
commit
251a4a9d73
47 files changed
+24148
-7
No files matched your search
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* Prometheus adapter. Config: url; optional: token (a bearer token, when something in front of Prometheus asks for one), insecure.
|
||||
*
|
||||
* Reading only: Prometheus's version, every scrape target and whether its last scrape worked (a node exporter that stopped answering), and the alerts that Prometheus's own rules have
|
||||
* pending or firing. That makes Prometheus a source: whatever you already measure there is seen here too. Nothing is sent to Prometheus but GET requests for these.
|
||||
*
|
||||
* Endpoints and field names are from Prometheus's published HTTP API (`/api/v1/status/buildinfo`, `/api/v1/targets`, `/api/v1/alerts`) and have not been checked against a live server
|
||||
* here; every field is read defensively. Alerts that Grafana manages itself are not in Prometheus's list and are not seen.
|
||||
*/
|
||||
import { requestJson } from "../httpJson.js";
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
|
||||
export interface PrometheusConfig {
|
||||
url: string;
|
||||
token?: string;
|
||||
insecure?: boolean;
|
||||
}
|
||||
|
||||
export interface PrometheusTarget {
|
||||
job: string;
|
||||
instance: string;
|
||||
health: "up" | "down" | "unknown";
|
||||
/** Why the last scrape failed. */
|
||||
error: string | null;
|
||||
lastScrape: string | null;
|
||||
}
|
||||
|
||||
export interface PrometheusAlert {
|
||||
name: string;
|
||||
/** From the alert's own `severity` label, lower-case; null when it has none. */
|
||||
severity: string | null;
|
||||
state: "firing" | "pending";
|
||||
summary: string | null;
|
||||
instance: string | null;
|
||||
activeAt: string | null;
|
||||
}
|
||||
|
||||
export interface PrometheusRead {
|
||||
version: string | null;
|
||||
/** Null when the targets could not be asked, which is not the same as there being none. */
|
||||
targets: PrometheusTarget[] | null;
|
||||
alerts: PrometheusAlert[] | null;
|
||||
}
|
||||
|
||||
export interface PrometheusAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(): Promise<PrometheusRead>;
|
||||
}
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
|
||||
const iso = (v: unknown): string | null => {
|
||||
const s = str(v);
|
||||
return s && Number.isFinite(Date.parse(s)) ? new Date(s).toISOString() : null;
|
||||
};
|
||||
|
||||
/** Pure: the version from /status/buildinfo. */
|
||||
export function parseBuildInfo(raw: unknown): string | null {
|
||||
return str(obj(obj(raw).data).version);
|
||||
}
|
||||
|
||||
/** Pure: the active targets from /targets; null when the answer has no list in it. */
|
||||
export function parseTargets(raw: unknown): PrometheusTarget[] | null {
|
||||
const list = obj(obj(raw).data).activeTargets;
|
||||
if (!Array.isArray(list)) return null;
|
||||
const out: PrometheusTarget[] = [];
|
||||
for (const item of list) {
|
||||
const t = obj(item);
|
||||
const labels = obj(t.labels);
|
||||
const job = str(labels.job) ?? str(t.scrapePool);
|
||||
if (!job) continue;
|
||||
const health = str(t.health)?.toLowerCase();
|
||||
out.push({
|
||||
job,
|
||||
instance: str(labels.instance) ?? str(t.scrapeUrl) ?? "?",
|
||||
health: health === "up" || health === "down" ? health : "unknown",
|
||||
error: str(t.lastError),
|
||||
lastScrape: iso(t.lastScrape),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: the alerts from /alerts; null when the answer has no list in it. */
|
||||
export function parseAlerts(raw: unknown): PrometheusAlert[] | null {
|
||||
const list = obj(obj(raw).data).alerts;
|
||||
if (!Array.isArray(list)) return null;
|
||||
const out: PrometheusAlert[] = [];
|
||||
for (const item of list) {
|
||||
const a = obj(item);
|
||||
const labels = obj(a.labels);
|
||||
const annotations = obj(a.annotations);
|
||||
const name = str(labels.alertname);
|
||||
const state = str(a.state)?.toLowerCase();
|
||||
if (!name || (state !== "firing" && state !== "pending")) continue;
|
||||
out.push({ name, severity: str(labels.severity)?.toLowerCase() ?? null, state, summary: str(annotations.summary) ?? str(annotations.description) ?? str(annotations.message), instance: str(labels.instance), activeAt: iso(a.activeAt) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function createPrometheusAdapter(config: PrometheusConfig): PrometheusAdapter {
|
||||
const base = String(config.url ?? "").replace(/\/+$/, "");
|
||||
const insecure = config.insecure === true;
|
||||
const headers: Record<string, string> = config.token ? { Authorization: `Bearer ${config.token}` } : {};
|
||||
|
||||
async function get(path: string): Promise<unknown> {
|
||||
const res = await requestJson(`${base}${path}`, { headers, insecure });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Prometheus did not accept the request (HTTP ${res.status}) — something in front of it wants a token`);
|
||||
if (res.status === 404) throw new Error("Prometheus has no API at this address — check the address and port (9090)");
|
||||
if (res.status < 200 || res.status >= 300) throw new Error(`Prometheus error: HTTP ${res.status}`);
|
||||
if (res.json === null) throw new Error("Prometheus answered with something that isn't JSON — is the address its own?");
|
||||
return res.json;
|
||||
}
|
||||
|
||||
async function read(): Promise<PrometheusRead> {
|
||||
const version = parseBuildInfo(await get("/api/v1/status/buildinfo"));
|
||||
// The rest is worth having but not worth failing for.
|
||||
const soft = async <T>(path: string, parse: (raw: unknown) => T): Promise<T | null> => {
|
||||
try {
|
||||
return parse(await get(path));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
const [targets, alerts] = await Promise.all([soft("/api/v1/targets?state=active", parseTargets), soft("/api/v1/alerts", parseAlerts)]);
|
||||
return { version, targets, alerts };
|
||||
}
|
||||
|
||||
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
await get("/api/v1/status/buildinfo");
|
||||
return { ok: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
return withDiagLogging("prometheus", { ping, read });
|
||||
}
|
||||
Reference in new issue
Block a user