Prometheus, Immich and Paperless-ngx integrations

Three read-only integrations, each with a monitoring page, alerts, maintenance
silencing and held-when-unreadable handling.

Prometheus: scrape targets that stop answering (5 minutes, one alert per job)
and alerts that Prometheus's own rules fire (critical when labelled so).
Immich: disk under the library filling, failed background jobs, counts.
Paperless-ngx: worker/database/Redis in error, documents that failed to
import, index and sanity check, disk. Immich and Paperless also in /metrics.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-24 22:33:17 +02:00
1 parent aebe98bf15
commit 251a4a9d73
47 files changed
+24148 -7

No files matched your search

+265
View File
@@ -0,0 +1,265 @@
/**
* Prometheus: reading what its API says (targets, alerts), what is stored and how long a target has been down, what is wrong, the health pass, the alerts, the page's route.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-prometheus-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, prometheusState } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/prometheus/adapter.js");
const pm = await import("../src/services/prometheusSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { prometheusRouter } = await import("../src/routes/prometheus.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const NOW = new Date("2026-10-20T12:00:00Z");
const ago = (min: number, from: Date = NOW) => new Date(from.getTime() - min * MIN);
type Read = import("../src/integrations/prometheus/adapter.js").PrometheusRead;
type Target = import("../src/integrations/prometheus/adapter.js").PrometheusTarget;
const target = (job: string, instance: string, health: Target["health"] = "up", error: string | null = null): Target => ({ job, instance, health, error, lastScrape: NOW.toISOString() });
const alert = (name: string, extra: Partial<import("../src/integrations/prometheus/adapter.js").PrometheusAlert> = {}) => ({ name, severity: "warning", state: "firing" as const, summary: null, instance: null, activeAt: NOW.toISOString(), ...extra });
const read = (extra: Partial<Read> = {}): Read => ({ version: "3.1.0", targets: [], alerts: [], ...extra });
describe("what Prometheus's API says", () => {
test("the version", () => {
assert.equal(adapter.parseBuildInfo({ status: "success", data: { version: "3.1.0" } }), "3.1.0");
assert.equal(adapter.parseBuildInfo(null), null);
});
test("targets, with their health and why a scrape failed; an answer without a list is not 'none'", () => {
const t = adapter.parseTargets({
data: {
activeTargets: [
{ labels: { job: "node", instance: "jackie:9100" }, health: "up", lastError: "", lastScrape: "2026-10-20T11:59:50Z" },
{ labels: { job: "node", instance: "nova:9100" }, health: "down", lastError: "connection refused", lastScrape: "2026-10-20T11:59:50Z" },
{ labels: { instance: "x" }, scrapePool: "pool", health: "weird" },
{ labels: {} },
],
},
})!;
assert.deepEqual(t.map((x) => [x.job, x.instance, x.health, x.error]), [["node", "jackie:9100", "up", null], ["node", "nova:9100", "down", "connection refused"], ["pool", "x", "unknown", null]]);
assert.equal(adapter.parseTargets({ status: "error" }), null);
assert.deepEqual(adapter.parseTargets({ data: { activeTargets: [] } }), []);
});
test("alerts: firing and pending, with severity and the summary", () => {
const a = adapter.parseAlerts({
data: {
alerts: [
{ labels: { alertname: "DiskFull", severity: "Critical", instance: "jackie:9100" }, annotations: { summary: "Disk is 95% full" }, state: "firing", activeAt: "2026-10-20T11:00:00Z" },
{ labels: { alertname: "Slow" }, annotations: { description: "slow" }, state: "pending" },
{ labels: { alertname: "Odd" }, state: "inactive" },
{ labels: {}, state: "firing" },
],
},
})!;
assert.deepEqual(a.map((x) => [x.name, x.severity, x.state, x.summary, x.instance]), [["DiskFull", "critical", "firing", "Disk is 95% full", "jackie:9100"], ["Slow", null, "pending", "slow", null]]);
assert.equal(adapter.parseAlerts({}), null);
});
describe("the adapter against a Prometheus that answers", () => {
let server: Server;
let url = "";
const seen: (string | undefined)[] = [];
before(async () => {
server = createHttpServer((req, res) => {
seen.push(req.headers.authorization);
const send = (code: number, body: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(body));
};
const path = (req.url ?? "").split("?")[0];
if (path === "/api/v1/status/buildinfo") return send(200, { data: { version: "3.1.0" } });
if (path === "/api/v1/targets") return send(200, { data: { activeTargets: [{ labels: { job: "node", instance: "a" }, health: "down", lastError: "timeout" }] } });
if (path === "/api/v1/alerts") return send(500, {}); // the rest is worth having, not worth failing for
send(404, {});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
});
after(() => server.close());
test("it reads, and an alert list that cannot be read is null rather than empty", async () => {
const r = await adapter.createPrometheusAdapter({ url: `${url}/` }).read();
assert.equal(r.version, "3.1.0");
assert.equal(r.targets?.length, 1);
assert.equal(r.alerts, null);
assert.ok(seen.every((s) => s === undefined), "no token, none sent");
});
test("a token is sent when there is one; a dead address is said", async () => {
await adapter.createPrometheusAdapter({ url, token: "t0k" }).ping();
assert.equal(seen.at(-1), "Bearer t0k");
assert.equal((await adapter.createPrometheusAdapter({ url: "http://127.0.0.1:1" }).ping()).ok, false);
});
});
});
describe("what is stored", () => {
let id = 0;
before(async () => {
id = (await db.insert(integrations).values({ type: "prometheus", name: "Metrics", baseUrl: "http://10.9.9.5:9090", config: JSON.stringify({ url: "http://10.9.9.5:9090" }) }).returning({ id: integrations.id }))[0].id;
});
const state = async () => (await db.select().from(prometheusState).where(eq(prometheusState.integrationId, id)))[0];
test("targets are counted by job, and a target that stays down keeps the time it was first seen down", async () => {
await pm.storeRead(id, read({ targets: [target("node", "a"), target("node", "b", "down", "refused"), target("cadvisor", "c")] }), ago(10));
await pm.storeRead(id, read({ targets: [target("node", "a"), target("node", "b", "down", "refused"), target("cadvisor", "c")] }), NOW);
const s = await state();
assert.deepEqual(pm.jobsOf(s), [{ job: "cadvisor", up: 1, down: 0 }, { job: "node", up: 1, down: 1 }]);
assert.equal(pm.downOf(s)?.[0].since, ago(10).toISOString(), "since the first look, not the last");
// It comes back and goes down again: the clock starts over.
await pm.storeRead(id, read({ targets: [target("node", "b")] }), NOW);
assert.deepEqual(pm.downOf(await state()), []);
await pm.storeRead(id, read({ targets: [target("node", "b", "down")] }), new Date(NOW.getTime() + MIN));
assert.equal(pm.downOf(await state())?.[0].since, new Date(NOW.getTime() + MIN).toISOString());
});
test("what could not be read stays null, not empty", async () => {
await pm.storeRead(id, read({ targets: null, alerts: null }), NOW);
const s = await state();
assert.deepEqual([pm.jobsOf(s), pm.downOf(s), pm.alertsOf(s)], [null, null, null]);
});
test("a read that failed is said, and what was stored is left alone", async () => {
const ok = await pm.syncPrometheus(id, new Date(), async () => read({ version: "3.2.0" }));
assert.equal(ok.ok, true);
const failed = await pm.syncPrometheus(id, new Date(), async () => {
throw new Error("connection refused");
});
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
assert.equal(failed.at, ok.at);
assert.equal((await state()).version, "3.2.0");
assert.equal(pm.isPrometheusFresh((await pm.prometheusSyncStatuses())[id], Date.now()), false);
});
});
describe("what is wrong", () => {
const row = (r: { jobs?: unknown; down?: unknown; alerts?: unknown }) => ({ jobs: JSON.stringify(r.jobs ?? []), down: JSON.stringify(r.down ?? []), alerts: JSON.stringify(r.alerts ?? []) });
const keys = (r: ReturnType<typeof row>) => pm.prometheusProblems(1, "Metrics", r, NOW).map((p) => `${p.key}=${p.severity}`);
test("a target is a problem only once it has been down five minutes; the job's targets are one alert", () => {
const jobs = [{ job: "node", up: 1, down: 3 }];
const d = (inst: string, mins: number) => ({ job: "node", instance: inst, error: "connection refused", since: ago(mins).toISOString() });
assert.deepEqual(keys(row({ jobs, down: [d("a", 4)] })), [], "one failed scrape is not news");
assert.deepEqual(keys(row({ jobs, down: [d("a", 6)] })), ["prometheus-target:1:node=warning"]);
const p = pm.prometheusProblems(1, "Metrics", row({ jobs, down: [d("a", 9), d("b", 9), d("c", 9), d("e", 1)] }), NOW);
assert.equal(p.length, 1);
assert.match(p[0].message, /3 of 4 “node” targets not answering \(a, b, c\) — connection refused/);
assert.equal(pm.prometheusProblems(1, "Metrics", row({ jobs, down: [d("a", 9), d("b", 9), d("c", 9)] }), NOW)[0].message.includes("and "), false);
});
test("firing alerts, one for each name; critical when any is; heartbeat alerts, pending and info ones are left out", () => {
const alerts = [
alert("DiskFull", { severity: "critical", instance: "jackie:9100", summary: "Disk is 95% full" }),
alert("DiskFull", { severity: "warning", instance: "nova:9100" }),
alert("HighLoad", { instance: "nova:9100" }),
alert("Watchdog", { severity: "none" }),
alert("Pending", { state: "pending" }),
alert("FYI", { severity: "info" }),
];
assert.deepEqual(keys(row({ alerts })), ["prometheus-alert:1:DiskFull=critical", "prometheus-alert:1:HighLoad=warning"]);
const p = pm.prometheusProblems(1, "Metrics", row({ alerts }), NOW);
assert.equal(p[0].message, "Metrics: DiskFull (2 firing) — Disk is 95% full");
assert.equal(p[1].message, "Metrics: HighLoad on nova:9100");
});
test("nothing read says nothing is wrong", () => {
assert.deepEqual(pm.prometheusProblems(1, "Metrics", { jobs: null, down: null, alerts: null }, NOW), []);
});
});
describe("the health pass and the page", () => {
let pmId = 0;
let app: Server;
let base = "";
const fresh = () => setInternalFlag("prometheusSync", JSON.stringify({ [pmId]: { at: new Date().toISOString(), ok: true, error: null } }));
const stale = () => setInternalFlag("prometheusSync", JSON.stringify({ [pmId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
pmId = (await db.insert(integrations).values({ type: "prometheus", name: "Prom", baseUrl: "http://10.9.9.4:9090", config: JSON.stringify({ url: "http://10.9.9.4:9090" }) }).returning({ id: integrations.id }))[0].id;
const t0 = new Date(Date.now() - 12 * MIN);
const r = read({ targets: [target("node", "jackie:9100", "down", "refused"), target("node", "nova:9100")], alerts: [alert("DiskFull", { severity: "critical", instance: "jackie:9100", summary: "Disk is 95% full" })] });
await pm.storeRead(pmId, r, t0);
await pm.storeRead(pmId, r, new Date());
await fresh();
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/prometheus", prometheusRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/prometheus`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
let { snapshot, held } = await collectSnapshot();
assert.deepEqual(snapshot.prometheus?.map((p) => p.integrationId), [pmId]);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const c = list.find((x) => x.key === `prometheus-alert:${pmId}:DiskFull`)!;
assert.deepEqual([c.severity, c.source], ["critical", `prometheus:${pmId}`]);
assert.equal(subjectOfConditionKey(c.key), `integration:${pmId}`, "maintenance on it silences its alerts");
assert.equal(subjectOfConditionKey(`prometheus-target:${pmId}:node`), `integration:${pmId}`);
assert.ok(list.some((x) => x.key === `prometheus-target:${pmId}:node`));
await stale();
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.prometheus ?? [], held.has(`prometheus:${pmId}`)], [[], true]);
await fresh();
});
test("the Alerts page shows them in the Monitoring kind, with where to look", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const a = report.alerts.find((x) => x.id === `monitoring:prometheus-alert:${pmId}:DiskFull`)!;
assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["critical", "monitoring", "Prometheus", "/prometheus", "integration", pmId]);
assert.ok(report.alerts.some((x) => x.id === `monitoring:prometheus-target:${pmId}:node`));
});
test("anyone signed in sees each Prometheus, its jobs, what is down and what fires; an old reading says nothing is wrong", async () => {
const body = (await (await fetch(base)).json()) as { prometheuses: { id: number; fresh: boolean; state: { jobs: { job: string; up: number; down: number }[]; down: unknown[]; alerts: unknown[] } | null; problems: unknown[] }[] };
const p = body.prometheuses.find((x) => x.id === pmId)!;
assert.equal(p.fresh, true);
assert.deepEqual(p.state?.jobs, [{ job: "node", up: 1, down: 1 }]);
assert.equal(p.state?.down.length, 1);
assert.equal(p.problems.length, 2);
await stale();
const old = (await (await fetch(base)).json()) as typeof body;
assert.deepEqual(old.prometheuses.find((x) => x.id === pmId)!.problems, []);
await fresh();
});
});