Prometheus, Immich and Paperless-ngx integrations
Three read-only integrations, each with a monitoring page, alerts, maintenance silencing and held-when-unreadable handling. Prometheus: scrape targets that stop answering (5 minutes, one alert per job) and alerts that Prometheus's own rules fire (critical when labelled so). Immich: disk under the library filling, failed background jobs, counts. Paperless-ngx: worker/database/Redis in error, documents that failed to import, index and sanity check, disk. Immich and Paperless also in /metrics. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aebe98bf15
commit
251a4a9d73
47 files changed
+24148
-7
No files matched your search
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* Keeps what Prometheus says up to date: every two minutes each Prometheus integration is read (its scrape targets and whether they answer, and the alerts its rules have firing)
|
||||
* and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Prometheus.
|
||||
*
|
||||
* Prometheus is where the things you already measure live — a node exporter on each host, a service's own metrics — so this makes them part of the one list of what is wrong:
|
||||
* a **target that has stopped answering** (for five minutes, so one failed scrape is not news) and **an alert that Prometheus's rules are firing**.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, prometheusState } from "../db/schema.js";
|
||||
import { createPrometheusAdapter, type PrometheusAlert, type PrometheusRead } from "../integrations/prometheus/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 2 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 10 * 60_000;
|
||||
/** A target must have been down this long before it is a problem; one failed scrape is not. */
|
||||
export const TARGET_DOWN_AFTER_MS = 5 * 60_000;
|
||||
const STATUS_FLAG = "prometheusSync";
|
||||
|
||||
export interface PrometheusSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface JobCount {
|
||||
job: string;
|
||||
up: number;
|
||||
down: number;
|
||||
}
|
||||
export interface DownTarget {
|
||||
job: string;
|
||||
instance: string;
|
||||
error: string | null;
|
||||
/** When it was first seen down, carried over from one read to the next. */
|
||||
since: string;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, PrometheusSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, PrometheusSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function prometheusSyncStatuses(): Promise<Record<number, PrometheusSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isPrometheusFresh = (status: PrometheusSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof prometheusState.$inferSelect;
|
||||
|
||||
/** Null stays null: a list that could not be read is not an empty one. */
|
||||
const parse = <T>(text: string | null): T[] | null => {
|
||||
if (text === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(text);
|
||||
return Array.isArray(v) ? (v as T[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
export const jobsOf = (row: Pick<StateRow, "jobs">) => parse<JobCount>(row.jobs);
|
||||
export const downOf = (row: Pick<StateRow, "down">) => parse<DownTarget>(row.down);
|
||||
export const alertsOf = (row: Pick<StateRow, "alerts">) => parse<PrometheusAlert>(row.alerts);
|
||||
|
||||
/** Writes down one read, replacing the last; a target that was down before keeps the time it was first seen down. */
|
||||
export async function storeRead(integrationId: number, r: PrometheusRead, now: Date = new Date()): Promise<void> {
|
||||
const [before] = await db.select({ down: prometheusState.down }).from(prometheusState).where(eq(prometheusState.integrationId, integrationId)).limit(1);
|
||||
const earlier = new Map((before ? downOf(before) ?? [] : []).map((d) => [`${d.job}\u0000${d.instance}`, d.since]));
|
||||
let jobs: JobCount[] | null = null;
|
||||
let down: DownTarget[] | null = null;
|
||||
if (r.targets) {
|
||||
const counts = new Map<string, JobCount>();
|
||||
down = [];
|
||||
for (const t of r.targets) {
|
||||
const c = counts.get(t.job) ?? { job: t.job, up: 0, down: 0 };
|
||||
if (t.health === "down") {
|
||||
c.down += 1;
|
||||
down.push({ job: t.job, instance: t.instance, error: t.error, since: earlier.get(`${t.job}\u0000${t.instance}`) ?? now.toISOString() });
|
||||
} else if (t.health === "up") c.up += 1;
|
||||
counts.set(t.job, c);
|
||||
}
|
||||
jobs = [...counts.values()].sort((a, b) => a.job.localeCompare(b.job));
|
||||
}
|
||||
const values = { version: r.version, jobs: jobs && JSON.stringify(jobs), down: down && JSON.stringify(down), alerts: r.alerts && JSON.stringify(r.alerts), updatedAt: now.toISOString() };
|
||||
await db.insert(prometheusState).values({ integrationId, ...values }).onConflictDoUpdate({ target: prometheusState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one Prometheus and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncPrometheus(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<PrometheusRead>): Promise<PrometheusSyncStatus> {
|
||||
let status: PrometheusSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "prometheus") throw new Error("not a Prometheus integration");
|
||||
const result = await (read ? read(loaded.config) : createPrometheusAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[prometheus] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[prometheus] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllPrometheus(now: Date = new Date()): Promise<Record<number, PrometheusSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "prometheus"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, PrometheusSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncPrometheus(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetPrometheus(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface PrometheusProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
/** Alerts that are firing by design (a heartbeat to prove the pipeline works) and so say nothing. */
|
||||
const BY_DESIGN = /^(watchdog|deadmansswitch|deadmanssnitch|infoinhibitor)$/i;
|
||||
|
||||
/** Pure: what is wrong, from what is stored. */
|
||||
export function prometheusProblems(id: number, name: string, s: Pick<StateRow, "jobs" | "down" | "alerts">, now: Date = new Date()): PrometheusProblem[] {
|
||||
const out: PrometheusProblem[] = [];
|
||||
const jobs = new Map((jobsOf(s) ?? []).map((j) => [j.job, j]));
|
||||
const byJob = new Map<string, DownTarget[]>();
|
||||
for (const d of downOf(s) ?? []) {
|
||||
if (now.getTime() - Date.parse(d.since) < TARGET_DOWN_AFTER_MS) continue;
|
||||
byJob.set(d.job, [...(byJob.get(d.job) ?? []), d]);
|
||||
}
|
||||
for (const [job, list] of byJob) {
|
||||
const c = jobs.get(job);
|
||||
const total = c ? c.up + c.down : list.length;
|
||||
const who = list.slice(0, 3).map((d) => d.instance).join(", ") + (list.length > 3 ? ` and ${list.length - 3} more` : "");
|
||||
const why = list[0].error ? ` — ${list[0].error.length > 120 ? `${list[0].error.slice(0, 117)}…` : list[0].error}` : "";
|
||||
out.push({ key: `prometheus-target:${id}:${job}`, severity: "warning", message: `${name}: ${list.length} of ${total} “${job}” target${total === 1 ? "" : "s"} not answering (${who})${why}` });
|
||||
}
|
||||
const firing = new Map<string, PrometheusAlert[]>();
|
||||
for (const a of alertsOf(s) ?? []) {
|
||||
if (a.state !== "firing" || BY_DESIGN.test(a.name) || a.severity === "info" || a.severity === "none") continue;
|
||||
firing.set(a.name, [...(firing.get(a.name) ?? []), a]);
|
||||
}
|
||||
for (const [alertname, list] of firing) {
|
||||
const critical = list.some((a) => a.severity === "critical" || a.severity === "page");
|
||||
const first = list.find((a) => a.summary) ?? list[0];
|
||||
out.push({
|
||||
key: `prometheus-alert:${id}:${alertname}`,
|
||||
severity: critical ? "critical" : "warning",
|
||||
message: `${name}: ${alertname}${list.length > 1 ? ` (${list.length} firing)` : first.instance ? ` on ${first.instance}` : ""}${first.summary ? ` — ${first.summary}` : ""}`,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initPrometheusSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllPrometheus().catch((err) => console.error("[prometheus] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 15_000);
|
||||
console.log(`[prometheus] reading Prometheus every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
Reference in new issue
Block a user