/** * NetBox: reading what its API says (status, counts, the addresses in pages), which servers are not in it, what is stored, what is wrong, the health pass, the alerts, the page's * route and /metrics. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { createServer as createHttpServer, type Server } from "node:http"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-netbox-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers, integrations, netboxState } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const adapter = await import("../src/integrations/netbox/adapter.js"); const nb = await import("../src/services/netboxSync.js"); const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js"); const { subjectOfConditionKey } = await import("../src/services/maintenance.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { netboxRouter } = await import("../src/routes/netbox.js"); const { buildMetrics } = await import("../src/services/metrics.js"); const { setInternalFlag } = await import("../src/services/settingsStore.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); const MIN = 60_000; type Read = import("../src/integrations/netbox/adapter.js").NetBoxRead; const read = (extra: Partial = {}): Read => ({ version: "4.2.5", workersRunning: 1, devices: 12, virtualMachines: 30, prefixes: 8, ipAddresses: 4, ips: ["10.0.0.5", "10.0.0.6", "10.0.0.7", "fd00::1"], ...extra }); describe("what NetBox's API says", () => { test("the status: version and workers; the workers are null when it does not say", () => { assert.deepEqual(adapter.parseStatus({ "netbox-version": "4.2.5", "rq-workers-running": 2 }), { version: "4.2.5", workersRunning: 2 }); assert.deepEqual(adapter.parseStatus({ "netbox-version": "4.2.5", "rq-workers-running": 0 }), { version: "4.2.5", workersRunning: 0 }); assert.deepEqual(adapter.parseStatus({}), { version: null, workersRunning: null }); }); test("counts, and addresses without their prefix length", () => { assert.equal(adapter.parseCount({ count: 12, results: [] }), 12); assert.equal(adapter.parseCount({}), null); assert.deepEqual(adapter.parseAddresses({ results: [{ address: "10.0.0.5/24" }, { address: "FD00::1/64" }, { address: "" }, {}] }), ["10.0.0.5", "fd00::1"]); }); test("a v2 token (nbt_…) goes as a Bearer token, an older one as Token", () => { assert.equal(adapter.authorization("abc123"), "Token abc123"); assert.equal(adapter.authorization("nbt_key.secret"), "Bearer nbt_key.secret"); }); describe("the adapter against a NetBox that answers", () => { let server: Server; let url = ""; const seen: string[] = []; const TOTAL = 1500; before(async () => { server = createHttpServer((req, res) => { const u = new URL(req.url ?? "", "http://x"); seen.push(`${req.headers.authorization} ${u.pathname}${u.search}`); const send = (code: number, body: unknown) => { res.writeHead(code, { "content-type": "application/json" }); res.end(JSON.stringify(body)); }; if (req.headers.authorization !== "Token tok") return send(403, {}); if (u.pathname === "/api/status/") return send(200, { "netbox-version": "4.2.5", "rq-workers-running": 0 }); if (u.pathname === "/api/dcim/devices/") return send(200, { count: 12, results: [] }); if (u.pathname === "/api/virtualization/virtual-machines/") return send(500, {}); // the rest is worth having, not worth failing for if (u.pathname === "/api/ipam/prefixes/") return send(200, { count: 8, results: [] }); if (u.pathname === "/api/ipam/ip-addresses/") { if (u.searchParams.get("limit") === "1") return send(200, { count: TOTAL, results: [] }); const offset = Number(u.searchParams.get("offset") ?? 0); const limit = Number(u.searchParams.get("limit") ?? 50); const results = Array.from({ length: Math.max(0, Math.min(limit, TOTAL - offset)) }, (_, i) => ({ address: `10.1.${Math.floor((offset + i) / 250)}.${((offset + i) % 250) + 1}/24` })); return send(200, { count: TOTAL, results }); } send(404, {}); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; }); after(() => server.close()); test("it reads the counts, the workers and every address in pages; a count that cannot be read is null", async () => { const r = await adapter.createNetBoxAdapter({ url: `${url}/`, token: "tok" }).read(); assert.deepEqual([r.version, r.workersRunning, r.devices, r.virtualMachines, r.prefixes, r.ipAddresses], ["4.2.5", 0, 12, null, 8, 1500]); assert.equal(r.ips?.length, 1500, "both pages"); assert.ok(r.ips?.includes("10.1.0.1") && r.ips?.includes("10.1.5.250")); const pages = seen.filter((s) => s.includes("brief=1")).map((s) => /offset=(\d+)/.exec(s)![1]); assert.deepEqual(pages, ["0", "1000"]); }); test("a refused token and a dead address are said plainly", async () => { const bad = await adapter.createNetBoxAdapter({ url, token: "nope" }).ping(); assert.equal(bad.ok, false); assert.match(bad.error ?? "", /did not accept the token/); assert.equal((await adapter.createNetBoxAdapter({ url, token: "tok" }).ping()).ok, true); assert.equal((await adapter.createNetBoxAdapter({ url: "http://127.0.0.1:1", token: "tok" }).ping()).ok, false); }); }); }); describe("which servers are not in NetBox", () => { const s = (id: number, name: string, ips: string[]) => ({ id, name, ips }); test("a server none of whose addresses are in NetBox is listed; one with any address in it is not", () => { const out = nb.serversMissingFromNetBox([s(1, "web", ["10.0.0.5", "172.17.0.1"]), s(2, "nas", ["10.0.0.9"]), s(3, "pi", ["10.0.0.7"])], ["10.0.0.5", "10.0.0.7"]); assert.deepEqual(out.map((x) => x.name), ["nas"]); }); test("a server that reports no usable address is not judged: loopback, link-local and nothing are nothing to look for", () => { const out = nb.serversMissingFromNetBox([s(1, "a", []), s(2, "b", ["127.0.0.1", "169.254.3.4", "fe80::1", "::1"]), s(3, "c", ["10.0.0.99"])], ["10.0.0.5"]); assert.deepEqual(out.map((x) => x.name), ["c"]); }); test("an address that is the same on every host (Docker's bridge) is taken out before it is weighed, and the order is by name", () => { const excluded = (ip: string) => ip.startsWith("172.17."); const out = nb.serversMissingFromNetBox([s(1, "zeta", ["172.17.0.1"]), s(2, "beta", ["172.17.0.1", "10.0.0.2"]), s(3, "alpha", ["10.0.0.1"])], ["10.0.0.2"], excluded); assert.deepEqual(out.map((x) => [x.name, x.ips]), [["alpha", ["10.0.0.1"]]], "zeta has only the bridge address, beta is in NetBox"); }); test("case and spaces do not matter", () => { assert.deepEqual(nb.serversMissingFromNetBox([s(1, "v6", [" FD00::1 "])], ["fd00::1"]), []); }); }); describe("what is stored", () => { let id = 0; before(async () => { id = (await db.insert(integrations).values({ type: "netbox", name: "IPAM", baseUrl: "http://10.9.7.1", config: JSON.stringify({ url: "http://10.9.7.1", token: "tok" }) }).returning({ id: integrations.id }))[0].id; }); const state = async () => (await db.select().from(netboxState).where(eq(netboxState.integrationId, id)))[0]; test("the state is replaced at each read; addresses that could not be listed stay null, not empty", async () => { await nb.storeRead(id, read(), new Date()); assert.deepEqual(nb.ipsOf(await state()), ["10.0.0.5", "10.0.0.6", "10.0.0.7", "fd00::1"]); await nb.storeRead(id, read({ ips: null, workersRunning: null }), new Date()); const s = await state(); assert.deepEqual([nb.ipsOf(s), s.workersRunning], [null, null]); }); test("a read that failed is said, and what was stored is left alone", async () => { const ok = await nb.syncNetBox(id, new Date(), async () => read({ version: "4.2.6" })); assert.equal(ok.ok, true); const failed = await nb.syncNetBox(id, new Date(), async () => { throw new Error("connection refused"); }); assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]); assert.equal(failed.at, ok.at); assert.equal((await state()).version, "4.2.6"); assert.equal(nb.isNetBoxFresh((await nb.netboxSyncStatuses())[id], Date.now()), false); }); }); describe("what is wrong", () => { test("no worker running is a warning; one running, or not said, is nothing", () => { assert.deepEqual(nb.netboxProblems(1, "IPAM", { workersRunning: 0 }).map((p) => [p.key, p.severity]), [["netbox-workers:1", "warning"]]); assert.match(nb.netboxProblems(1, "IPAM", { workersRunning: 0 })[0].message, /no background worker is running/); assert.deepEqual(nb.netboxProblems(1, "IPAM", { workersRunning: 1 }), []); assert.deepEqual(nb.netboxProblems(1, "IPAM", { workersRunning: null }), []); }); }); describe("the health pass, the page and /metrics", () => { let nbId = 0; let app: Server; let base = ""; const ids: Record = {}; const fresh = () => setInternalFlag("netboxSync", JSON.stringify({ [nbId]: { at: new Date().toISOString(), ok: true, error: null } })); const stale = () => setInternalFlag("netboxSync", JSON.stringify({ [nbId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } })); before(async () => { await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]); nbId = (await db.insert(integrations).values({ type: "netbox", name: "NetBox", baseUrl: "http://10.9.7.2", config: JSON.stringify({ url: "http://10.9.7.2", token: "tok" }) }).returning({ id: integrations.id }))[0].id; const mk = async (name: string, ips: string[]) => (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, ipAddresses: JSON.stringify(ips) }).returning({ id: servers.id }))[0].id; ids.web = await mk("web", ["10.0.0.5"]); ids.nas = await mk("nas", ["10.0.0.99"]); await nb.storeRead(nbId, read({ workersRunning: 0, ipAddresses: 4 }), new Date()); await fresh(); const a = express(); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: "viewer" } }; next(); }); a.use("/api/netbox", netboxRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/netbox`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("the health pass judges it from what is stored, and holds it when the last read is old", async () => { let { snapshot, held } = await collectSnapshot(); assert.deepEqual(snapshot.netbox?.map((n) => n.integrationId), [nbId]); const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true }); const c = list.find((x) => x.key === `netbox-workers:${nbId}`)!; assert.deepEqual([c.severity, c.source], ["warning", `netbox:${nbId}`]); assert.equal(subjectOfConditionKey(c.key), `integration:${nbId}`, "maintenance on it silences its alerts"); await stale(); ({ snapshot, held } = await collectSnapshot()); assert.deepEqual([snapshot.netbox ?? [], held.has(`netbox:${nbId}`)], [[], true]); await fresh(); }); test("the Alerts page shows it under Services, with where to look", async () => { invalidateAlertsCache(); const report = await getAlerts(true); const a = report.alerts.find((x) => x.id === `services:netbox-workers:${nbId}`)!; assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["warning", "services", "NetBox", "/netbox", "integration", nbId]); }); test("anyone signed in sees each NetBox, the counts and which servers are not in it; an old reading says nothing is wrong", async () => { const body = (await (await fetch(base)).json()) as { netboxes: { id: number; fresh: boolean; state: { workersRunning: number; missing: { name: string }[] | null; truncated: boolean } | null; problems: unknown[] }[] }; const n = body.netboxes.find((x) => x.id === nbId)!; assert.deepEqual([n.fresh, n.state?.workersRunning, n.state?.missing?.map((m) => m.name), n.state?.truncated], [true, 0, ["nas"], false]); assert.equal(n.problems.length, 1); await stale(); const old = (await (await fetch(base)).json()) as typeof body; assert.deepEqual(old.netboxes.find((x) => x.id === nbId)!.problems, []); await fresh(); }); test("when NetBox has more addresses than were read, the page says what is missing may be in the rest; when they could not be listed, nothing is claimed", async () => { await nb.storeRead(nbId, read({ ipAddresses: 9000 }), new Date()); const body = (await (await fetch(base)).json()) as { netboxes: { id: number; state: { truncated: boolean; missing: unknown } }[] }; assert.equal(body.netboxes.find((x) => x.id === nbId)!.state.truncated, true); await nb.storeRead(nbId, read({ ips: null }), new Date()); const none = (await (await fetch(base)).json()) as typeof body; assert.equal(none.netboxes.find((x) => x.id === nbId)!.state.missing, null); await nb.storeRead(nbId, read({ workersRunning: 0, ipAddresses: 4 }), new Date()); }); test("/metrics has the workers and the counts — only while the reading is recent", async () => { let text = await buildMetrics(new Date()); assert.match(text, /homelab_netbox_workers\{name="NetBox"\} 0/); assert.match(text, /homelab_netbox_devices\{name="NetBox"\} 12/); assert.match(text, /homelab_netbox_ip_addresses\{name="NetBox"\} 4/); await stale(); text = await buildMetrics(new Date()); assert.doesNotMatch(text, /homelab_netbox/); }); });