Three read-only integrations, each with a monitoring page, alerts, maintenance silencing and held-when-unreadable handling. Prometheus: scrape targets that stop answering (5 minutes, one alert per job) and alerts that Prometheus's own rules fire (critical when labelled so). Immich: disk under the library filling, failed background jobs, counts. Paperless-ngx: worker/database/Redis in error, documents that failed to import, index and sanity check, disk. Immich and Paperless also in /metrics. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
141 lines
5.8 KiB
TypeScript
141 lines
5.8 KiB
TypeScript
/**
|
|
* Prometheus adapter. Config: url; optional: token (a bearer token, when something in front of Prometheus asks for one), insecure.
|
|
*
|
|
* Reading only: Prometheus's version, every scrape target and whether its last scrape worked (a node exporter that stopped answering), and the alerts that Prometheus's own rules have
|
|
* pending or firing. That makes Prometheus a source: whatever you already measure there is seen here too. Nothing is sent to Prometheus but GET requests for these.
|
|
*
|
|
* Endpoints and field names are from Prometheus's published HTTP API (`/api/v1/status/buildinfo`, `/api/v1/targets`, `/api/v1/alerts`) and have not been checked against a live server
|
|
* here; every field is read defensively. Alerts that Grafana manages itself are not in Prometheus's list and are not seen.
|
|
*/
|
|
import { requestJson } from "../httpJson.js";
|
|
import { withDiagLogging } from "../../services/diagLog.js";
|
|
|
|
export interface PrometheusConfig {
|
|
url: string;
|
|
token?: string;
|
|
insecure?: boolean;
|
|
}
|
|
|
|
export interface PrometheusTarget {
|
|
job: string;
|
|
instance: string;
|
|
health: "up" | "down" | "unknown";
|
|
/** Why the last scrape failed. */
|
|
error: string | null;
|
|
lastScrape: string | null;
|
|
}
|
|
|
|
export interface PrometheusAlert {
|
|
name: string;
|
|
/** From the alert's own `severity` label, lower-case; null when it has none. */
|
|
severity: string | null;
|
|
state: "firing" | "pending";
|
|
summary: string | null;
|
|
instance: string | null;
|
|
activeAt: string | null;
|
|
}
|
|
|
|
export interface PrometheusRead {
|
|
version: string | null;
|
|
/** Null when the targets could not be asked, which is not the same as there being none. */
|
|
targets: PrometheusTarget[] | null;
|
|
alerts: PrometheusAlert[] | null;
|
|
}
|
|
|
|
export interface PrometheusAdapter {
|
|
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
|
read(): Promise<PrometheusRead>;
|
|
}
|
|
|
|
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
|
|
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
|
|
const iso = (v: unknown): string | null => {
|
|
const s = str(v);
|
|
return s && Number.isFinite(Date.parse(s)) ? new Date(s).toISOString() : null;
|
|
};
|
|
|
|
/** Pure: the version from /status/buildinfo. */
|
|
export function parseBuildInfo(raw: unknown): string | null {
|
|
return str(obj(obj(raw).data).version);
|
|
}
|
|
|
|
/** Pure: the active targets from /targets; null when the answer has no list in it. */
|
|
export function parseTargets(raw: unknown): PrometheusTarget[] | null {
|
|
const list = obj(obj(raw).data).activeTargets;
|
|
if (!Array.isArray(list)) return null;
|
|
const out: PrometheusTarget[] = [];
|
|
for (const item of list) {
|
|
const t = obj(item);
|
|
const labels = obj(t.labels);
|
|
const job = str(labels.job) ?? str(t.scrapePool);
|
|
if (!job) continue;
|
|
const health = str(t.health)?.toLowerCase();
|
|
out.push({
|
|
job,
|
|
instance: str(labels.instance) ?? str(t.scrapeUrl) ?? "?",
|
|
health: health === "up" || health === "down" ? health : "unknown",
|
|
error: str(t.lastError),
|
|
lastScrape: iso(t.lastScrape),
|
|
});
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/** Pure: the alerts from /alerts; null when the answer has no list in it. */
|
|
export function parseAlerts(raw: unknown): PrometheusAlert[] | null {
|
|
const list = obj(obj(raw).data).alerts;
|
|
if (!Array.isArray(list)) return null;
|
|
const out: PrometheusAlert[] = [];
|
|
for (const item of list) {
|
|
const a = obj(item);
|
|
const labels = obj(a.labels);
|
|
const annotations = obj(a.annotations);
|
|
const name = str(labels.alertname);
|
|
const state = str(a.state)?.toLowerCase();
|
|
if (!name || (state !== "firing" && state !== "pending")) continue;
|
|
out.push({ name, severity: str(labels.severity)?.toLowerCase() ?? null, state, summary: str(annotations.summary) ?? str(annotations.description) ?? str(annotations.message), instance: str(labels.instance), activeAt: iso(a.activeAt) });
|
|
}
|
|
return out;
|
|
}
|
|
|
|
export function createPrometheusAdapter(config: PrometheusConfig): PrometheusAdapter {
|
|
const base = String(config.url ?? "").replace(/\/+$/, "");
|
|
const insecure = config.insecure === true;
|
|
const headers: Record<string, string> = config.token ? { Authorization: `Bearer ${config.token}` } : {};
|
|
|
|
async function get(path: string): Promise<unknown> {
|
|
const res = await requestJson(`${base}${path}`, { headers, insecure });
|
|
if (res.status === 401 || res.status === 403) throw new Error(`Prometheus did not accept the request (HTTP ${res.status}) — something in front of it wants a token`);
|
|
if (res.status === 404) throw new Error("Prometheus has no API at this address — check the address and port (9090)");
|
|
if (res.status < 200 || res.status >= 300) throw new Error(`Prometheus error: HTTP ${res.status}`);
|
|
if (res.json === null) throw new Error("Prometheus answered with something that isn't JSON — is the address its own?");
|
|
return res.json;
|
|
}
|
|
|
|
async function read(): Promise<PrometheusRead> {
|
|
const version = parseBuildInfo(await get("/api/v1/status/buildinfo"));
|
|
// The rest is worth having but not worth failing for.
|
|
const soft = async <T>(path: string, parse: (raw: unknown) => T): Promise<T | null> => {
|
|
try {
|
|
return parse(await get(path));
|
|
} catch {
|
|
return null;
|
|
}
|
|
};
|
|
const [targets, alerts] = await Promise.all([soft("/api/v1/targets?state=active", parseTargets), soft("/api/v1/alerts", parseAlerts)]);
|
|
return { version, targets, alerts };
|
|
}
|
|
|
|
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
|
const start = Date.now();
|
|
try {
|
|
await get("/api/v1/status/buildinfo");
|
|
return { ok: true, latencyMs: Date.now() - start };
|
|
} catch (err) {
|
|
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
|
}
|
|
}
|
|
|
|
return withDiagLogging("prometheus", { ping, read });
|
|
}
|