/** * Dependencies between servers and services: what is folded under what when something is down, and the routes that say what needs what. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; import type { Server } from "node:http"; const dir = mkdtempSync(join(tmpdir(), "hm-deps-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers, integrations, auditLog } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { buildEdges, coveredNodes, wouldLoop, integrationCausedBy, HEALTH_STATE_FLAG } = await import("../src/services/dependencies.js"); const { setInternalFlag } = await import("../src/services/settingsStore.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { trackIntegrationHealth } = await import("../src/services/integrationHealthMonitor.js"); const { dependenciesRouter } = await import("../src/routes/dependencies.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); const everything = () => true; const edge = (fromType: "server" | "integration", fromId: number, toType: "server" | "integration", toId: number) => ({ fromType, fromId, toType, toId }); describe("who is down because of whom", () => { test("a node is explained by the root of what it needs, however far up; things that are up in between don't matter", () => { const edges = buildEdges(new Map(), [edge("integration", 1, "server", 1), edge("integration", 2, "integration", 1), edge("server", 2, "integration", 2)], everything); const covered = coveredNodes(new Set(["server:1", "integration:1", "integration:2", "server:2"]), edges); assert.deepEqual([...covered.entries()].sort(), [["integration:1", "server:1"], ["integration:2", "server:1"], ["server:2", "server:1"]]); // the middle one is up: the one above it still explains the one below const skip = coveredNodes(new Set(["server:1", "server:2"]), edges); assert.deepEqual([...skip.entries()], [["server:2", "server:1"]]); }); test("the 'runs on' links of servers are part of the graph", () => { const edges = buildEdges(new Map([[2, { parentId: 1, auto: false }]]), [edge("integration", 1, "server", 2)], everything); const covered = coveredNodes(new Set(["server:1", "integration:1"]), edges); assert.equal(covered.get("integration:1"), "server:1"); }); test("something that needs two things is explained by whichever is down; with none down it stands alone", () => { const edges = buildEdges(new Map(), [edge("server", 1, "server", 2), edge("server", 1, "integration", 1)], everything); assert.equal(coveredNodes(new Set(["server:1", "integration:1"]), edges).get("server:1"), "integration:1"); assert.equal(coveredNodes(new Set(["server:1"]), edges).size, 0); }); test("a loop explains nothing, and edges to things that are gone are ignored", () => { const loop = buildEdges(new Map(), [edge("server", 1, "server", 2), edge("server", 2, "server", 1)], everything); assert.equal(coveredNodes(new Set(["server:1", "server:2"]), loop).size, 0); const gone = buildEdges(new Map(), [edge("server", 1, "server", 99)], (k) => k !== "server:99"); assert.equal(gone.size, 0); }); test("a new dependency that would close a loop is seen, including through 'runs on'", () => { const edges = buildEdges(new Map([[2, { parentId: 1, auto: true }]]), [edge("integration", 1, "server", 2)], everything); assert.equal(wouldLoop(edges, "server:1", "integration:1"), true, "the integration needs server 2, which runs on server 1"); assert.equal(wouldLoop(edges, "server:1", "server:1"), true); assert.equal(wouldLoop(edges, "integration:1", "server:1"), false, "already needs it, indirectly: not a loop"); }); }); describe("through the routes, the alerts and the notifications", () => { let app: Server; let base = ""; let who = "operator"; let box = 0; let other = 0; let gitea = 0; let semaphore = 0; before(async () => { await db.insert(users).values([ { oidcSub: "operator", name: "Olga", role: "operator" }, { oidcSub: "viewer", name: "Vic", role: "viewer" }, ]); const hoursAgo = (n: number) => new Date(Date.now() - n * 3600_000).toISOString(); const made = await db .insert(servers) .values([ { name: "box", apiTokenHash: "b", apiTokenPrefix: "b", lastSeenAt: hoursAgo(3) }, { name: "other", apiTokenHash: "o", apiTokenPrefix: "o", lastSeenAt: new Date().toISOString() }, ]) .returning({ id: servers.id }); box = made[0].id; other = made[1].id; const ints = await db .insert(integrations) .values([ { type: "gitea", name: "Gitea", baseUrl: "https://git.lab", config: "{}" }, { type: "semaphore", name: "Semaphore", baseUrl: "https://sem.lab", config: "{}" }, ]) .returning({ id: integrations.id }); gitea = ints[0].id; semaphore = ints[1].id; const a = express(); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/dependencies", dependenciesRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/dependencies`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); let firstId = 0; test("an operator says what needs what; it is listed with names", async () => { const res = await send("POST", "/", edge("integration", gitea, "server", box)); assert.equal(res.status, 201); firstId = (await res.json()).id; assert.equal((await send("POST", "/", edge("integration", semaphore, "integration", gitea))).status, 201); const list = await (await send("GET", "/")).json(); assert.deepEqual(list.dependencies.map((d: { from: { name: string }; to: { name: string } }) => `${d.from.name} > ${d.to.name}`).sort(), ["Gitea > box", "Semaphore > Gitea"]); assert.ok(list.nodes.some((n: { key: string; name: string }) => n.key === `server:${box}` && n.name === "box")); }); test("it can't be nonsense: not itself, not twice, not a loop, not something that isn't there, not by a viewer", async () => { assert.equal((await send("POST", "/", edge("server", box, "server", box))).status, 400); assert.equal((await send("POST", "/", edge("integration", gitea, "server", box))).status, 409, "twice"); const loop = await send("POST", "/", edge("server", box, "integration", semaphore)); assert.equal(loop.status, 409, "the server would wait for the thing that waits for it"); assert.match((await loop.json()).message, /wait for each other/); assert.equal((await send("POST", "/", edge("server", 99999, "server", box))).status, 400); assert.equal((await send("POST", "/", { fromType: "dns_provider", fromId: 1, toType: "server", toId: box })).status, 400); who = "viewer"; assert.equal((await send("POST", "/", edge("server", other, "server", box))).status, 403); assert.equal((await send("GET", "/")).status, 200, "everyone signed in can look"); who = "operator"; }); test("when the server is down and both services are failing, only the server is announced; the services are held back under it", async () => { await trackIntegrationHealth("gitea", false); await trackIntegrationHealth("semaphore", false); invalidateAlertsCache(); const report = await getAlerts(true); const offline = report.alerts.find((a) => a.category === "offline" && a.message.startsWith("box"))!; assert.equal(offline.silenced, false); assert.deepEqual(offline.affects, ["Gitea", "Semaphore"]); const failing = report.alerts.filter((a) => a.category === "integration"); assert.equal(failing.length, 2); for (const f of failing) { assert.deepEqual(f.causedBy, { name: "box" }); assert.equal(f.silenced, true); } assert.equal(failing.find((f) => f.id.includes("gitea"))!.affects, undefined, "only the root says what it takes with it"); assert.equal(report.counts.critical + report.counts.warning, 1); assert.equal(report.counts.grouped, 2); }); test("when the server is back, Gitea stands alone and Semaphore is held back under Gitea", async () => { await db.update(servers).set({ lastSeenAt: new Date().toISOString() }).where(eq(servers.id, box)); invalidateAlertsCache(); const report = await getAlerts(true); const failing = report.alerts.filter((a) => a.category === "integration"); assert.equal(failing.find((f) => f.id.includes("gitea"))!.causedBy, undefined); assert.deepEqual(failing.find((f) => f.id.includes("semaphore"))!.causedBy, { name: "Gitea" }); }); test("a kind of service is folded only if every enabled instance of it is explained", async () => { const [second] = await db.insert(integrations).values({ type: "gitea", name: "Gitea 2", baseUrl: "https://git2.lab", config: "{}" }).returning({ id: integrations.id }); await db.update(servers).set({ lastSeenAt: new Date(Date.now() - 3 * 3600_000).toISOString() }).where(eq(servers.id, box)); invalidateAlertsCache(); const report = await getAlerts(true); assert.equal(report.alerts.find((a) => a.category === "integration" && a.id.includes("gitea"))!.causedBy, undefined, "the second one doesn't need the box"); await db.delete(integrations).where(eq(integrations.id, second.id)); }); test("a failure that crosses its threshold while the server it needs is down isn't announced; one that doesn't need it is", async () => { assert.equal(await integrationCausedBy("gitea", ["gitea"]), null, "nothing is known to be down yet"); await setInternalFlag(HEALTH_STATE_FLAG, JSON.stringify({ [`offline:server:${box}`]: { message: "box is offline", source: "server" } })); assert.deepEqual(await integrationCausedBy("gitea", ["gitea"]), { key: `server:${box}`, name: "box" }); assert.deepEqual(await integrationCausedBy("semaphore", ["gitea", "semaphore"]), { key: `server:${box}`, name: "box" }, "through Gitea"); assert.equal(await integrationCausedBy("tailscale", ["tailscale"]), null, "an unrelated kind"); await setInternalFlag(HEALTH_STATE_FLAG, JSON.stringify({ [`offline:server:${other}`]: { message: "other is offline", source: "server" } })); assert.equal(await integrationCausedBy("gitea", ["gitea"]), null, "a different server being down explains nothing"); }); test("removing one works, is audited, and the services stand alone again", async () => { assert.equal((await send("DELETE", `/${firstId}`)).status, 204); assert.equal((await send("DELETE", `/${firstId}`)).status, 404); const actions = (await db.select().from(auditLog)).filter((e) => e.category === "dependency").map((e) => e.action); assert.deepEqual(actions.sort(), ["create", "create", "delete"]); invalidateAlertsCache(); const report = await getAlerts(true); assert.equal(report.alerts.find((a) => a.category === "integration" && a.id.includes("gitea"))!.causedBy, undefined); }); });