Three read-only integrations, each with a monitoring page, alerts, maintenance silencing and held-when-unreadable handling. Prometheus: scrape targets that stop answering (5 minutes, one alert per job) and alerts that Prometheus's own rules fire (critical when labelled so). Immich: disk under the library filling, failed background jobs, counts. Paperless-ngx: worker/database/Redis in error, documents that failed to import, index and sanity check, disk. Immich and Paperless also in /metrics. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
266 lines
15 KiB
TypeScript
266 lines
15 KiB
TypeScript
/**
|
|
* Prometheus: reading what its API says (targets, alerts), what is stored and how long a target has been down, what is wrong, the health pass, the alerts, the page's route.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { createServer as createHttpServer, type Server } from "node:http";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { AddressInfo } from "node:net";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-prometheus-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const express = (await import("express")).default;
|
|
const { db } = await import("../src/db/client.js");
|
|
const { users, integrations, prometheusState } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const adapter = await import("../src/integrations/prometheus/adapter.js");
|
|
const pm = await import("../src/services/prometheusSync.js");
|
|
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
|
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
|
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
|
const { prometheusRouter } = await import("../src/routes/prometheus.js");
|
|
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
|
const { eq } = await import("drizzle-orm");
|
|
|
|
await runMigrations();
|
|
|
|
const MIN = 60_000;
|
|
const NOW = new Date("2026-10-20T12:00:00Z");
|
|
const ago = (min: number, from: Date = NOW) => new Date(from.getTime() - min * MIN);
|
|
|
|
type Read = import("../src/integrations/prometheus/adapter.js").PrometheusRead;
|
|
type Target = import("../src/integrations/prometheus/adapter.js").PrometheusTarget;
|
|
const target = (job: string, instance: string, health: Target["health"] = "up", error: string | null = null): Target => ({ job, instance, health, error, lastScrape: NOW.toISOString() });
|
|
const alert = (name: string, extra: Partial<import("../src/integrations/prometheus/adapter.js").PrometheusAlert> = {}) => ({ name, severity: "warning", state: "firing" as const, summary: null, instance: null, activeAt: NOW.toISOString(), ...extra });
|
|
const read = (extra: Partial<Read> = {}): Read => ({ version: "3.1.0", targets: [], alerts: [], ...extra });
|
|
|
|
describe("what Prometheus's API says", () => {
|
|
test("the version", () => {
|
|
assert.equal(adapter.parseBuildInfo({ status: "success", data: { version: "3.1.0" } }), "3.1.0");
|
|
assert.equal(adapter.parseBuildInfo(null), null);
|
|
});
|
|
|
|
test("targets, with their health and why a scrape failed; an answer without a list is not 'none'", () => {
|
|
const t = adapter.parseTargets({
|
|
data: {
|
|
activeTargets: [
|
|
{ labels: { job: "node", instance: "jackie:9100" }, health: "up", lastError: "", lastScrape: "2026-10-20T11:59:50Z" },
|
|
{ labels: { job: "node", instance: "nova:9100" }, health: "down", lastError: "connection refused", lastScrape: "2026-10-20T11:59:50Z" },
|
|
{ labels: { instance: "x" }, scrapePool: "pool", health: "weird" },
|
|
{ labels: {} },
|
|
],
|
|
},
|
|
})!;
|
|
assert.deepEqual(t.map((x) => [x.job, x.instance, x.health, x.error]), [["node", "jackie:9100", "up", null], ["node", "nova:9100", "down", "connection refused"], ["pool", "x", "unknown", null]]);
|
|
assert.equal(adapter.parseTargets({ status: "error" }), null);
|
|
assert.deepEqual(adapter.parseTargets({ data: { activeTargets: [] } }), []);
|
|
});
|
|
|
|
test("alerts: firing and pending, with severity and the summary", () => {
|
|
const a = adapter.parseAlerts({
|
|
data: {
|
|
alerts: [
|
|
{ labels: { alertname: "DiskFull", severity: "Critical", instance: "jackie:9100" }, annotations: { summary: "Disk is 95% full" }, state: "firing", activeAt: "2026-10-20T11:00:00Z" },
|
|
{ labels: { alertname: "Slow" }, annotations: { description: "slow" }, state: "pending" },
|
|
{ labels: { alertname: "Odd" }, state: "inactive" },
|
|
{ labels: {}, state: "firing" },
|
|
],
|
|
},
|
|
})!;
|
|
assert.deepEqual(a.map((x) => [x.name, x.severity, x.state, x.summary, x.instance]), [["DiskFull", "critical", "firing", "Disk is 95% full", "jackie:9100"], ["Slow", null, "pending", "slow", null]]);
|
|
assert.equal(adapter.parseAlerts({}), null);
|
|
});
|
|
|
|
describe("the adapter against a Prometheus that answers", () => {
|
|
let server: Server;
|
|
let url = "";
|
|
const seen: (string | undefined)[] = [];
|
|
before(async () => {
|
|
server = createHttpServer((req, res) => {
|
|
seen.push(req.headers.authorization);
|
|
const send = (code: number, body: unknown) => {
|
|
res.writeHead(code, { "content-type": "application/json" });
|
|
res.end(JSON.stringify(body));
|
|
};
|
|
const path = (req.url ?? "").split("?")[0];
|
|
if (path === "/api/v1/status/buildinfo") return send(200, { data: { version: "3.1.0" } });
|
|
if (path === "/api/v1/targets") return send(200, { data: { activeTargets: [{ labels: { job: "node", instance: "a" }, health: "down", lastError: "timeout" }] } });
|
|
if (path === "/api/v1/alerts") return send(500, {}); // the rest is worth having, not worth failing for
|
|
send(404, {});
|
|
});
|
|
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
|
});
|
|
after(() => server.close());
|
|
|
|
test("it reads, and an alert list that cannot be read is null rather than empty", async () => {
|
|
const r = await adapter.createPrometheusAdapter({ url: `${url}/` }).read();
|
|
assert.equal(r.version, "3.1.0");
|
|
assert.equal(r.targets?.length, 1);
|
|
assert.equal(r.alerts, null);
|
|
assert.ok(seen.every((s) => s === undefined), "no token, none sent");
|
|
});
|
|
|
|
test("a token is sent when there is one; a dead address is said", async () => {
|
|
await adapter.createPrometheusAdapter({ url, token: "t0k" }).ping();
|
|
assert.equal(seen.at(-1), "Bearer t0k");
|
|
assert.equal((await adapter.createPrometheusAdapter({ url: "http://127.0.0.1:1" }).ping()).ok, false);
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("what is stored", () => {
|
|
let id = 0;
|
|
before(async () => {
|
|
id = (await db.insert(integrations).values({ type: "prometheus", name: "Metrics", baseUrl: "http://10.9.9.5:9090", config: JSON.stringify({ url: "http://10.9.9.5:9090" }) }).returning({ id: integrations.id }))[0].id;
|
|
});
|
|
const state = async () => (await db.select().from(prometheusState).where(eq(prometheusState.integrationId, id)))[0];
|
|
|
|
test("targets are counted by job, and a target that stays down keeps the time it was first seen down", async () => {
|
|
await pm.storeRead(id, read({ targets: [target("node", "a"), target("node", "b", "down", "refused"), target("cadvisor", "c")] }), ago(10));
|
|
await pm.storeRead(id, read({ targets: [target("node", "a"), target("node", "b", "down", "refused"), target("cadvisor", "c")] }), NOW);
|
|
const s = await state();
|
|
assert.deepEqual(pm.jobsOf(s), [{ job: "cadvisor", up: 1, down: 0 }, { job: "node", up: 1, down: 1 }]);
|
|
assert.equal(pm.downOf(s)?.[0].since, ago(10).toISOString(), "since the first look, not the last");
|
|
// It comes back and goes down again: the clock starts over.
|
|
await pm.storeRead(id, read({ targets: [target("node", "b")] }), NOW);
|
|
assert.deepEqual(pm.downOf(await state()), []);
|
|
await pm.storeRead(id, read({ targets: [target("node", "b", "down")] }), new Date(NOW.getTime() + MIN));
|
|
assert.equal(pm.downOf(await state())?.[0].since, new Date(NOW.getTime() + MIN).toISOString());
|
|
});
|
|
|
|
test("what could not be read stays null, not empty", async () => {
|
|
await pm.storeRead(id, read({ targets: null, alerts: null }), NOW);
|
|
const s = await state();
|
|
assert.deepEqual([pm.jobsOf(s), pm.downOf(s), pm.alertsOf(s)], [null, null, null]);
|
|
});
|
|
|
|
test("a read that failed is said, and what was stored is left alone", async () => {
|
|
const ok = await pm.syncPrometheus(id, new Date(), async () => read({ version: "3.2.0" }));
|
|
assert.equal(ok.ok, true);
|
|
const failed = await pm.syncPrometheus(id, new Date(), async () => {
|
|
throw new Error("connection refused");
|
|
});
|
|
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
|
|
assert.equal(failed.at, ok.at);
|
|
assert.equal((await state()).version, "3.2.0");
|
|
assert.equal(pm.isPrometheusFresh((await pm.prometheusSyncStatuses())[id], Date.now()), false);
|
|
});
|
|
});
|
|
|
|
describe("what is wrong", () => {
|
|
const row = (r: { jobs?: unknown; down?: unknown; alerts?: unknown }) => ({ jobs: JSON.stringify(r.jobs ?? []), down: JSON.stringify(r.down ?? []), alerts: JSON.stringify(r.alerts ?? []) });
|
|
const keys = (r: ReturnType<typeof row>) => pm.prometheusProblems(1, "Metrics", r, NOW).map((p) => `${p.key}=${p.severity}`);
|
|
|
|
test("a target is a problem only once it has been down five minutes; the job's targets are one alert", () => {
|
|
const jobs = [{ job: "node", up: 1, down: 3 }];
|
|
const d = (inst: string, mins: number) => ({ job: "node", instance: inst, error: "connection refused", since: ago(mins).toISOString() });
|
|
assert.deepEqual(keys(row({ jobs, down: [d("a", 4)] })), [], "one failed scrape is not news");
|
|
assert.deepEqual(keys(row({ jobs, down: [d("a", 6)] })), ["prometheus-target:1:node=warning"]);
|
|
const p = pm.prometheusProblems(1, "Metrics", row({ jobs, down: [d("a", 9), d("b", 9), d("c", 9), d("e", 1)] }), NOW);
|
|
assert.equal(p.length, 1);
|
|
assert.match(p[0].message, /3 of 4 “node” targets not answering \(a, b, c\) — connection refused/);
|
|
assert.equal(pm.prometheusProblems(1, "Metrics", row({ jobs, down: [d("a", 9), d("b", 9), d("c", 9)] }), NOW)[0].message.includes("and "), false);
|
|
});
|
|
|
|
test("firing alerts, one for each name; critical when any is; heartbeat alerts, pending and info ones are left out", () => {
|
|
const alerts = [
|
|
alert("DiskFull", { severity: "critical", instance: "jackie:9100", summary: "Disk is 95% full" }),
|
|
alert("DiskFull", { severity: "warning", instance: "nova:9100" }),
|
|
alert("HighLoad", { instance: "nova:9100" }),
|
|
alert("Watchdog", { severity: "none" }),
|
|
alert("Pending", { state: "pending" }),
|
|
alert("FYI", { severity: "info" }),
|
|
];
|
|
assert.deepEqual(keys(row({ alerts })), ["prometheus-alert:1:DiskFull=critical", "prometheus-alert:1:HighLoad=warning"]);
|
|
const p = pm.prometheusProblems(1, "Metrics", row({ alerts }), NOW);
|
|
assert.equal(p[0].message, "Metrics: DiskFull (2 firing) — Disk is 95% full");
|
|
assert.equal(p[1].message, "Metrics: HighLoad on nova:9100");
|
|
});
|
|
|
|
test("nothing read says nothing is wrong", () => {
|
|
assert.deepEqual(pm.prometheusProblems(1, "Metrics", { jobs: null, down: null, alerts: null }, NOW), []);
|
|
});
|
|
});
|
|
|
|
describe("the health pass and the page", () => {
|
|
let pmId = 0;
|
|
let app: Server;
|
|
let base = "";
|
|
const fresh = () => setInternalFlag("prometheusSync", JSON.stringify({ [pmId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
|
const stale = () => setInternalFlag("prometheusSync", JSON.stringify({ [pmId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
|
|
|
|
before(async () => {
|
|
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
|
pmId = (await db.insert(integrations).values({ type: "prometheus", name: "Prom", baseUrl: "http://10.9.9.4:9090", config: JSON.stringify({ url: "http://10.9.9.4:9090" }) }).returning({ id: integrations.id }))[0].id;
|
|
const t0 = new Date(Date.now() - 12 * MIN);
|
|
const r = read({ targets: [target("node", "jackie:9100", "down", "refused"), target("node", "nova:9100")], alerts: [alert("DiskFull", { severity: "critical", instance: "jackie:9100", summary: "Disk is 95% full" })] });
|
|
await pm.storeRead(pmId, r, t0);
|
|
await pm.storeRead(pmId, r, new Date());
|
|
await fresh();
|
|
|
|
const a = express();
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
|
next();
|
|
});
|
|
a.use("/api/prometheus", prometheusRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/prometheus`;
|
|
});
|
|
after(() => {
|
|
app.close();
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows keeps the database file locked until the process exits.
|
|
}
|
|
});
|
|
|
|
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
|
|
let { snapshot, held } = await collectSnapshot();
|
|
assert.deepEqual(snapshot.prometheus?.map((p) => p.integrationId), [pmId]);
|
|
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
|
const c = list.find((x) => x.key === `prometheus-alert:${pmId}:DiskFull`)!;
|
|
assert.deepEqual([c.severity, c.source], ["critical", `prometheus:${pmId}`]);
|
|
assert.equal(subjectOfConditionKey(c.key), `integration:${pmId}`, "maintenance on it silences its alerts");
|
|
assert.equal(subjectOfConditionKey(`prometheus-target:${pmId}:node`), `integration:${pmId}`);
|
|
assert.ok(list.some((x) => x.key === `prometheus-target:${pmId}:node`));
|
|
await stale();
|
|
({ snapshot, held } = await collectSnapshot());
|
|
assert.deepEqual([snapshot.prometheus ?? [], held.has(`prometheus:${pmId}`)], [[], true]);
|
|
await fresh();
|
|
});
|
|
|
|
test("the Alerts page shows them in the Monitoring kind, with where to look", async () => {
|
|
invalidateAlertsCache();
|
|
const report = await getAlerts(true);
|
|
const a = report.alerts.find((x) => x.id === `monitoring:prometheus-alert:${pmId}:DiskFull`)!;
|
|
assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["critical", "monitoring", "Prometheus", "/prometheus", "integration", pmId]);
|
|
assert.ok(report.alerts.some((x) => x.id === `monitoring:prometheus-target:${pmId}:node`));
|
|
});
|
|
|
|
test("anyone signed in sees each Prometheus, its jobs, what is down and what fires; an old reading says nothing is wrong", async () => {
|
|
const body = (await (await fetch(base)).json()) as { prometheuses: { id: number; fresh: boolean; state: { jobs: { job: string; up: number; down: number }[]; down: unknown[]; alerts: unknown[] } | null; problems: unknown[] }[] };
|
|
const p = body.prometheuses.find((x) => x.id === pmId)!;
|
|
assert.equal(p.fresh, true);
|
|
assert.deepEqual(p.state?.jobs, [{ job: "node", up: 1, down: 1 }]);
|
|
assert.equal(p.state?.down.length, 1);
|
|
assert.equal(p.problems.length, 2);
|
|
await stale();
|
|
const old = (await (await fetch(base)).json()) as typeof body;
|
|
assert.deepEqual(old.prometheuses.find((x) => x.id === pmId)!.problems, []);
|
|
await fresh();
|
|
});
|
|
});
|