/** * Prometheus: reading what its API says (targets, alerts), what is stored and how long a target has been down, what is wrong, the health pass, the alerts, the page's route. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { createServer as createHttpServer, type Server } from "node:http"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-prometheus-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, integrations, prometheusState } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const adapter = await import("../src/integrations/prometheus/adapter.js"); const pm = await import("../src/services/prometheusSync.js"); const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js"); const { subjectOfConditionKey } = await import("../src/services/maintenance.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { prometheusRouter } = await import("../src/routes/prometheus.js"); const { setInternalFlag } = await import("../src/services/settingsStore.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); const MIN = 60_000; const NOW = new Date("2026-10-20T12:00:00Z"); const ago = (min: number, from: Date = NOW) => new Date(from.getTime() - min * MIN); type Read = import("../src/integrations/prometheus/adapter.js").PrometheusRead; type Target = import("../src/integrations/prometheus/adapter.js").PrometheusTarget; const target = (job: string, instance: string, health: Target["health"] = "up", error: string | null = null): Target => ({ job, instance, health, error, lastScrape: NOW.toISOString() }); const alert = (name: string, extra: Partial = {}) => ({ name, severity: "warning", state: "firing" as const, summary: null, instance: null, activeAt: NOW.toISOString(), ...extra }); const read = (extra: Partial = {}): Read => ({ version: "3.1.0", targets: [], alerts: [], ...extra }); describe("what Prometheus's API says", () => { test("the version", () => { assert.equal(adapter.parseBuildInfo({ status: "success", data: { version: "3.1.0" } }), "3.1.0"); assert.equal(adapter.parseBuildInfo(null), null); }); test("targets, with their health and why a scrape failed; an answer without a list is not 'none'", () => { const t = adapter.parseTargets({ data: { activeTargets: [ { labels: { job: "node", instance: "jackie:9100" }, health: "up", lastError: "", lastScrape: "2026-10-20T11:59:50Z" }, { labels: { job: "node", instance: "nova:9100" }, health: "down", lastError: "connection refused", lastScrape: "2026-10-20T11:59:50Z" }, { labels: { instance: "x" }, scrapePool: "pool", health: "weird" }, { labels: {} }, ], }, })!; assert.deepEqual(t.map((x) => [x.job, x.instance, x.health, x.error]), [["node", "jackie:9100", "up", null], ["node", "nova:9100", "down", "connection refused"], ["pool", "x", "unknown", null]]); assert.equal(adapter.parseTargets({ status: "error" }), null); assert.deepEqual(adapter.parseTargets({ data: { activeTargets: [] } }), []); }); test("alerts: firing and pending, with severity and the summary", () => { const a = adapter.parseAlerts({ data: { alerts: [ { labels: { alertname: "DiskFull", severity: "Critical", instance: "jackie:9100" }, annotations: { summary: "Disk is 95% full" }, state: "firing", activeAt: "2026-10-20T11:00:00Z" }, { labels: { alertname: "Slow" }, annotations: { description: "slow" }, state: "pending" }, { labels: { alertname: "Odd" }, state: "inactive" }, { labels: {}, state: "firing" }, ], }, })!; assert.deepEqual(a.map((x) => [x.name, x.severity, x.state, x.summary, x.instance]), [["DiskFull", "critical", "firing", "Disk is 95% full", "jackie:9100"], ["Slow", null, "pending", "slow", null]]); assert.equal(adapter.parseAlerts({}), null); }); describe("the adapter against a Prometheus that answers", () => { let server: Server; let url = ""; const seen: (string | undefined)[] = []; before(async () => { server = createHttpServer((req, res) => { seen.push(req.headers.authorization); const send = (code: number, body: unknown) => { res.writeHead(code, { "content-type": "application/json" }); res.end(JSON.stringify(body)); }; const path = (req.url ?? "").split("?")[0]; if (path === "/api/v1/status/buildinfo") return send(200, { data: { version: "3.1.0" } }); if (path === "/api/v1/targets") return send(200, { data: { activeTargets: [{ labels: { job: "node", instance: "a" }, health: "down", lastError: "timeout" }] } }); if (path === "/api/v1/alerts") return send(500, {}); // the rest is worth having, not worth failing for send(404, {}); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; }); after(() => server.close()); test("it reads, and an alert list that cannot be read is null rather than empty", async () => { const r = await adapter.createPrometheusAdapter({ url: `${url}/` }).read(); assert.equal(r.version, "3.1.0"); assert.equal(r.targets?.length, 1); assert.equal(r.alerts, null); assert.ok(seen.every((s) => s === undefined), "no token, none sent"); }); test("a token is sent when there is one; a dead address is said", async () => { await adapter.createPrometheusAdapter({ url, token: "t0k" }).ping(); assert.equal(seen.at(-1), "Bearer t0k"); assert.equal((await adapter.createPrometheusAdapter({ url: "http://127.0.0.1:1" }).ping()).ok, false); }); }); }); describe("what is stored", () => { let id = 0; before(async () => { id = (await db.insert(integrations).values({ type: "prometheus", name: "Metrics", baseUrl: "http://10.9.9.5:9090", config: JSON.stringify({ url: "http://10.9.9.5:9090" }) }).returning({ id: integrations.id }))[0].id; }); const state = async () => (await db.select().from(prometheusState).where(eq(prometheusState.integrationId, id)))[0]; test("targets are counted by job, and a target that stays down keeps the time it was first seen down", async () => { await pm.storeRead(id, read({ targets: [target("node", "a"), target("node", "b", "down", "refused"), target("cadvisor", "c")] }), ago(10)); await pm.storeRead(id, read({ targets: [target("node", "a"), target("node", "b", "down", "refused"), target("cadvisor", "c")] }), NOW); const s = await state(); assert.deepEqual(pm.jobsOf(s), [{ job: "cadvisor", up: 1, down: 0 }, { job: "node", up: 1, down: 1 }]); assert.equal(pm.downOf(s)?.[0].since, ago(10).toISOString(), "since the first look, not the last"); // It comes back and goes down again: the clock starts over. await pm.storeRead(id, read({ targets: [target("node", "b")] }), NOW); assert.deepEqual(pm.downOf(await state()), []); await pm.storeRead(id, read({ targets: [target("node", "b", "down")] }), new Date(NOW.getTime() + MIN)); assert.equal(pm.downOf(await state())?.[0].since, new Date(NOW.getTime() + MIN).toISOString()); }); test("what could not be read stays null, not empty", async () => { await pm.storeRead(id, read({ targets: null, alerts: null }), NOW); const s = await state(); assert.deepEqual([pm.jobsOf(s), pm.downOf(s), pm.alertsOf(s)], [null, null, null]); }); test("a read that failed is said, and what was stored is left alone", async () => { const ok = await pm.syncPrometheus(id, new Date(), async () => read({ version: "3.2.0" })); assert.equal(ok.ok, true); const failed = await pm.syncPrometheus(id, new Date(), async () => { throw new Error("connection refused"); }); assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]); assert.equal(failed.at, ok.at); assert.equal((await state()).version, "3.2.0"); assert.equal(pm.isPrometheusFresh((await pm.prometheusSyncStatuses())[id], Date.now()), false); }); }); describe("what is wrong", () => { const row = (r: { jobs?: unknown; down?: unknown; alerts?: unknown }) => ({ jobs: JSON.stringify(r.jobs ?? []), down: JSON.stringify(r.down ?? []), alerts: JSON.stringify(r.alerts ?? []) }); const keys = (r: ReturnType) => pm.prometheusProblems(1, "Metrics", r, NOW).map((p) => `${p.key}=${p.severity}`); test("a target is a problem only once it has been down five minutes; the job's targets are one alert", () => { const jobs = [{ job: "node", up: 1, down: 3 }]; const d = (inst: string, mins: number) => ({ job: "node", instance: inst, error: "connection refused", since: ago(mins).toISOString() }); assert.deepEqual(keys(row({ jobs, down: [d("a", 4)] })), [], "one failed scrape is not news"); assert.deepEqual(keys(row({ jobs, down: [d("a", 6)] })), ["prometheus-target:1:node=warning"]); const p = pm.prometheusProblems(1, "Metrics", row({ jobs, down: [d("a", 9), d("b", 9), d("c", 9), d("e", 1)] }), NOW); assert.equal(p.length, 1); assert.match(p[0].message, /3 of 4 “node” targets not answering \(a, b, c\) — connection refused/); assert.equal(pm.prometheusProblems(1, "Metrics", row({ jobs, down: [d("a", 9), d("b", 9), d("c", 9)] }), NOW)[0].message.includes("and "), false); }); test("firing alerts, one for each name; critical when any is; heartbeat alerts, pending and info ones are left out", () => { const alerts = [ alert("DiskFull", { severity: "critical", instance: "jackie:9100", summary: "Disk is 95% full" }), alert("DiskFull", { severity: "warning", instance: "nova:9100" }), alert("HighLoad", { instance: "nova:9100" }), alert("Watchdog", { severity: "none" }), alert("Pending", { state: "pending" }), alert("FYI", { severity: "info" }), ]; assert.deepEqual(keys(row({ alerts })), ["prometheus-alert:1:DiskFull=critical", "prometheus-alert:1:HighLoad=warning"]); const p = pm.prometheusProblems(1, "Metrics", row({ alerts }), NOW); assert.equal(p[0].message, "Metrics: DiskFull (2 firing) — Disk is 95% full"); assert.equal(p[1].message, "Metrics: HighLoad on nova:9100"); }); test("nothing read says nothing is wrong", () => { assert.deepEqual(pm.prometheusProblems(1, "Metrics", { jobs: null, down: null, alerts: null }, NOW), []); }); }); describe("the health pass and the page", () => { let pmId = 0; let app: Server; let base = ""; const fresh = () => setInternalFlag("prometheusSync", JSON.stringify({ [pmId]: { at: new Date().toISOString(), ok: true, error: null } })); const stale = () => setInternalFlag("prometheusSync", JSON.stringify({ [pmId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } })); before(async () => { await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]); pmId = (await db.insert(integrations).values({ type: "prometheus", name: "Prom", baseUrl: "http://10.9.9.4:9090", config: JSON.stringify({ url: "http://10.9.9.4:9090" }) }).returning({ id: integrations.id }))[0].id; const t0 = new Date(Date.now() - 12 * MIN); const r = read({ targets: [target("node", "jackie:9100", "down", "refused"), target("node", "nova:9100")], alerts: [alert("DiskFull", { severity: "critical", instance: "jackie:9100", summary: "Disk is 95% full" })] }); await pm.storeRead(pmId, r, t0); await pm.storeRead(pmId, r, new Date()); await fresh(); const a = express(); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: "viewer" } }; next(); }); a.use("/api/prometheus", prometheusRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/prometheus`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("the health pass judges it from what is stored, and holds it when the last read is old", async () => { let { snapshot, held } = await collectSnapshot(); assert.deepEqual(snapshot.prometheus?.map((p) => p.integrationId), [pmId]); const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true }); const c = list.find((x) => x.key === `prometheus-alert:${pmId}:DiskFull`)!; assert.deepEqual([c.severity, c.source], ["critical", `prometheus:${pmId}`]); assert.equal(subjectOfConditionKey(c.key), `integration:${pmId}`, "maintenance on it silences its alerts"); assert.equal(subjectOfConditionKey(`prometheus-target:${pmId}:node`), `integration:${pmId}`); assert.ok(list.some((x) => x.key === `prometheus-target:${pmId}:node`)); await stale(); ({ snapshot, held } = await collectSnapshot()); assert.deepEqual([snapshot.prometheus ?? [], held.has(`prometheus:${pmId}`)], [[], true]); await fresh(); }); test("the Alerts page shows them in the Monitoring kind, with where to look", async () => { invalidateAlertsCache(); const report = await getAlerts(true); const a = report.alerts.find((x) => x.id === `monitoring:prometheus-alert:${pmId}:DiskFull`)!; assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["critical", "monitoring", "Prometheus", "/prometheus", "integration", pmId]); assert.ok(report.alerts.some((x) => x.id === `monitoring:prometheus-target:${pmId}:node`)); }); test("anyone signed in sees each Prometheus, its jobs, what is down and what fires; an old reading says nothing is wrong", async () => { const body = (await (await fetch(base)).json()) as { prometheuses: { id: number; fresh: boolean; state: { jobs: { job: string; up: number; down: number }[]; down: unknown[]; alerts: unknown[] } | null; problems: unknown[] }[] }; const p = body.prometheuses.find((x) => x.id === pmId)!; assert.equal(p.fresh, true); assert.deepEqual(p.state?.jobs, [{ job: "node", up: 1, down: 1 }]); assert.equal(p.state?.down.length, 1); assert.equal(p.problems.length, 2); await stale(); const old = (await (await fetch(base)).json()) as typeof body; assert.deepEqual(old.prometheuses.find((x) => x.id === pmId)!.problems, []); await fresh(); }); });