/** * An update run: the maintenance window, the playbook, following it, waiting for the server to report again, and every way it can end. * Semaphore is a stand-in that answers as told. * * Run with: npm test --workspace server */ import { test, describe, before, after, beforeEach } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; import type { Server } from "node:http"; const dir = mkdtempSync(join(tmpdir(), "hm-updaterun-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers, updateRuns, auditLog, maintenanceWindows } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { startUpdateRun, advanceUpdateRuns, cancelUpdateRun, listUpdateRuns, setSemaphoreForTests, summarizeOutcome, UpdateRunError, HOST_WAIT_MS, MAX_POLL_ERRORS, RUN_LIMIT_MS } = await import("../src/services/updateRuns.js"); const { listActiveWindows } = await import("../src/services/maintenance.js"); const { getActivity } = await import("../src/services/serverActivity.js").then((m) => ({ getActivity: m.getServerActivity })); const { updateRunsRouter } = await import("../src/routes/updateRuns.js"); const { hashToken } = await import("../src/services/tokens.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); // ── a Semaphore that answers as told ── interface Fake { templates: { id: number; projectId: number; projectName: string; name: string; playbook: string; lastTask: null }[]; runs: { projectId: number; templateId: number; options?: { limit?: string; message?: string } }[]; task: { id: number; status: string; created: string | null; start: string | null; end: string | null }; refuseToStart: boolean; getTaskFails: boolean; } const fake: Fake = { templates: [], runs: [], task: { id: 41, status: "waiting", created: null, start: null, end: null }, refuseToStart: false, getTaskFails: false }; const resetFake = () => { fake.templates = [{ id: 7, projectId: 2, projectName: "Homelab", name: "Apply updates", playbook: "updates.yml", lastTask: null }]; fake.runs = []; fake.task = { id: 41, status: "waiting", created: null, start: null, end: null }; fake.refuseToStart = false; fake.getTaskFails = false; }; setSemaphoreForTests(async (integrationId) => integrationId === 9 ? { ping: async () => ({ ok: true }), listTemplatesWithStatus: async () => fake.templates, checkTemplates: async () => ({ templates: fake.templates, failedProjectIds: [] }), runTemplate: async (projectId, templateId, options) => { if (fake.refuseToStart) throw new Error("Semaphore is having a moment"); fake.runs.push({ projectId, templateId, options }); return { ...fake.task } as never; }, getTask: async () => { if (fake.getTaskFails) throw new Error("connection refused"); return { ...fake.task } as never; }, } : null, ); const T0 = new Date("2026-10-17T12:00:00.000Z"); const at = (minutes: number) => new Date(T0.getTime() + minutes * 60_000); const input = (serverId: number, extra: object = {}) => ({ serverId, integrationId: 9, projectId: 2, templateId: 7, limit: "elsa", windowMinutes: 90, actor: "Olle", ...extra }); const runRow = async (id: number) => (await db.select().from(updateRuns).where(eq(updateRuns.id, id)))[0]; describe("starting a run", () => { let serverId = 0; before(async () => { await db.insert(users).values([{ oidcSub: "admin", name: "Ada", role: "admin" }]); const [s] = await db .insert(servers) .values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: T0.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 12, security: 3, rebootRequired: false, reportedAt: T0.toISOString() }) }) .returning(); serverId = s.id; }); beforeEach(async () => { resetFake(); await db.delete(updateRuns); await db.delete(maintenanceWindows); }); after(() => { try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("it puts the server under maintenance, starts the playbook limited to that server, and writes the run down", async () => { const run = await startUpdateRun(input(serverId), T0); assert.equal(run.status, "running"); assert.equal(run.templateName, "Apply updates"); assert.equal(run.updatesBefore, 12); assert.equal(run.taskId, 41); assert.equal(run.windowCreated, true); assert.deepEqual(fake.runs.map((r) => [r.projectId, r.templateId, r.options?.limit]), [[2, 7, "elsa"]]); assert.match(fake.runs[0].options!.message!, /Update run for elsa, started in Homelab Manager by Olle \(limited to elsa\)/); const windows = await listActiveWindows(T0); assert.equal(windows.length, 1); assert.equal(windows[0].targetId, serverId); assert.equal(windows[0].reason, "Update run: Apply updates"); assert.equal(Date.parse(windows[0].endsAt) - T0.getTime(), 90 * 60_000); assert.equal(run.windowId, windows[0].id); }); test("it won't start without something to wait for, with another run going, or on a Semaphore or template that isn't there", async () => { const refused = async (promise: Promise, status: number, code: string) => { await assert.rejects(promise, (e: unknown) => e instanceof UpdateRunError && e.status === status && e.code === code, code); }; await refused(startUpdateRun(input(99999), T0), 404, "server_not_found"); const [never] = await db.insert(servers).values({ name: "ghost", apiTokenHash: hashToken("g"), apiTokenPrefix: "g" }).returning(); await refused(startUpdateRun(input(never.id), T0), 400, "never_reported"); await refused(startUpdateRun(input(serverId, { integrationId: 5 }), T0), 400, "no_semaphore"); await refused(startUpdateRun(input(serverId, { templateId: 999 }), T0), 404, "template_not_found"); assert.equal(fake.runs.length, 0, "nothing was started by any of those"); assert.equal((await listActiveWindows(T0)).length, 0, "and no quiet was left behind"); await startUpdateRun(input(serverId), T0); await refused(startUpdateRun(input(serverId), at(1)), 409, "already_running"); assert.equal(fake.runs.length, 1); }); test("if Semaphore won't start it the quiet is taken back, and no run is recorded", async () => { fake.refuseToStart = true; await assert.rejects(startUpdateRun(input(serverId), T0), (e: unknown) => e instanceof UpdateRunError && e.status === 502 && /Semaphore wouldn't start it/.test(e.message)); assert.equal((await db.select().from(updateRuns)).length, 0); assert.equal((await listActiveWindows(at(1))).length, 0, "the window it made is over"); }); test("a window that was already there is extended and left to its owner; it is not ended when the run is", async () => { await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "patch night", startedAt: T0.toISOString(), endsAt: at(30).toISOString(), createdBy: "Ada" }); const run = await startUpdateRun(input(serverId), at(1)); assert.equal(run.windowCreated, false); fake.task = { ...fake.task, status: "error" }; await advanceUpdateRuns(at(5)); assert.equal((await runRow(run.id)).status, "failed"); assert.equal((await listActiveWindows(at(6))).length, 1, "someone else's window still stands"); }); }); describe("following a run to the end", () => { let serverId = 0; before(async () => { const [s] = await db.insert(servers).values({ name: "anna", apiTokenHash: hashToken("tok2"), apiTokenPrefix: "t2", lastSeenAt: T0.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 12, security: 3, rebootRequired: false, reportedAt: T0.toISOString() }) }).returning(); serverId = s.id; }); beforeEach(async () => { resetFake(); await db.delete(updateRuns); await db.delete(maintenanceWindows); await db.update(servers).set({ lastSeenAt: T0.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 12, security: 3, rebootRequired: false, reportedAt: T0.toISOString() }) }).where(eq(servers.id, serverId)); }); const reportBack = async (minutes: number, count: number, reboot: boolean) => db.update(servers).set({ lastSeenAt: at(minutes).toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count, security: 0, rebootRequired: reboot, reportedAt: at(minutes).toISOString() }) }).where(eq(servers.id, serverId)); test("a good run: running, then waiting for the host, then done with what is left — and the window ends", async () => { const run = await startUpdateRun(input(serverId), T0); fake.task = { ...fake.task, status: "running" }; await advanceUpdateRuns(at(1)); let r = await runRow(run.id); assert.equal(r.status, "running"); assert.equal(r.taskStatus, "running"); fake.task = { ...fake.task, status: "success", end: at(8).toISOString() }; await advanceUpdateRuns(at(9)); r = await runRow(run.id); assert.equal(r.status, "waiting_for_host"); assert.equal(r.taskEndedAt, at(8).toISOString()); assert.equal((await listActiveWindows(at(9))).length, 1, "still quiet while it comes back"); await advanceUpdateRuns(at(12)); assert.equal((await runRow(run.id)).status, "waiting_for_host", "no report since the playbook ended"); await reportBack(7, 12, false); // a report from before the playbook finished doesn't count await advanceUpdateRuns(at(13)); assert.equal((await runRow(run.id)).status, "waiting_for_host"); await reportBack(14, 2, true); await advanceUpdateRuns(at(15)); r = await runRow(run.id); assert.equal(r.status, "done"); assert.equal(r.updatesBefore, 12); assert.equal(r.updatesAfter, 2); assert.equal(r.rebootRequired, true); assert.equal(r.message, "2 updates still waiting (10 updates installed); a restart is still needed."); assert.equal(r.finishedAt, at(15).toISOString()); assert.equal((await listActiveWindows(at(16))).length, 0, "the window it made is over"); assert.equal((await advanceUpdateRuns(at(17))), 0, "a finished run is left alone"); }); test("a playbook that fails ends the run and the window", async () => { const run = await startUpdateRun(input(serverId), T0); fake.task = { ...fake.task, status: "error" }; await advanceUpdateRuns(at(3)); const r = await runRow(run.id); assert.equal(r.status, "failed"); assert.match(r.message!, /The playbook failed in Semaphore \(run #41\)/); assert.equal((await listActiveWindows(at(4))).length, 0); for (const [status, words] of [["rejected", /rejected/], ["stopped", /stopped/]] as const) { await db.delete(updateRuns); const again = await startUpdateRun(input(serverId), at(10)); fake.task = { ...fake.task, status }; await advanceUpdateRuns(at(11)); assert.match((await runRow(again.id)).message!, words); } }); test("a Semaphore that stops answering ends it after ten tries; one answer in between starts the count again", async () => { const run = await startUpdateRun(input(serverId), T0); fake.getTaskFails = true; for (let i = 0; i < MAX_POLL_ERRORS - 1; i++) await advanceUpdateRuns(at(1 + i)); assert.equal((await runRow(run.id)).status, "running"); assert.equal((await runRow(run.id)).pollErrors, MAX_POLL_ERRORS - 1); fake.getTaskFails = false; fake.task = { ...fake.task, status: "running" }; await advanceUpdateRuns(at(20)); assert.equal((await runRow(run.id)).pollErrors, 0); fake.getTaskFails = true; for (let i = 0; i < MAX_POLL_ERRORS; i++) await advanceUpdateRuns(at(21 + i)); const r = await runRow(run.id); assert.equal(r.status, "failed"); assert.match(r.message!, /Lost contact with Semaphore/); }); test("a host that doesn't come back, or a run that never ends, is given up on, and the quiet ends so the alerts can speak", async () => { const run = await startUpdateRun(input(serverId), T0); fake.task = { ...fake.task, status: "success", end: at(5).toISOString() }; await advanceUpdateRuns(at(6)); await advanceUpdateRuns(at(5 + HOST_WAIT_MS / 60_000 - 1)); assert.equal((await runRow(run.id)).status, "waiting_for_host"); await advanceUpdateRuns(at(5 + HOST_WAIT_MS / 60_000 + 1)); const r = await runRow(run.id); assert.equal(r.status, "failed"); assert.match(r.message!, /hasn't reported again in 30 minutes/); assert.equal((await listActiveWindows(at(40))).length, 0); await db.delete(updateRuns); const slow = await startUpdateRun(input(serverId), at(100)); fake.task = { ...fake.task, status: "running" }; await advanceUpdateRuns(new Date(at(100).getTime() + RUN_LIMIT_MS + 60_000)); assert.match((await runRow(slow.id)).message!, /longer than three hours/); }); test("a server whose agent doesn't report updates is done as soon as it reports at all", async () => { await db.update(servers).set({ pendingUpdates: null }).where(eq(servers.id, serverId)); const run = await startUpdateRun(input(serverId), T0); fake.task = { ...fake.task, status: "success", end: at(5).toISOString() }; await advanceUpdateRuns(at(6)); await db.update(servers).set({ lastSeenAt: at(10).toISOString() }).where(eq(servers.id, serverId)); await advanceUpdateRuns(at(11)); const r = await runRow(run.id); assert.equal(r.status, "done"); assert.equal(r.updatesAfter, null); assert.match(r.message!, /doesn't say what is waiting; no restart is needed/); }); test("it can be stopped: the window ends, the Semaphore run is left alone, and a finished run can't be stopped again", async () => { const run = await startUpdateRun(input(serverId), T0); const stopped = await cancelUpdateRun(run.id, "Ada", at(2)); assert.equal(stopped!.status, "cancelled"); assert.match(stopped!.message!, /Stopped being followed by Ada/); assert.equal((await listActiveWindows(at(3))).length, 0); assert.equal(fake.runs.length, 1, "nothing was sent to Semaphore to stop it"); await assert.rejects(cancelUpdateRun(run.id, "Ada", at(3)), (e: unknown) => e instanceof UpdateRunError && e.status === 409); assert.equal(await cancelUpdateRun(99999, "Ada", at(3)), null); }); test("the story is on the server's history, and a run that was under way when the app stopped is picked up again", async () => { const run = await startUpdateRun(input(serverId), T0); fake.task = { ...fake.task, status: "error" }; await advanceUpdateRuns(at(3)); const events = await getActivity(serverId, at(5)); const titles = events.filter((e) => /Update run/.test(e.title)).map((e) => e.title); // (the tests before this one wrote their own lines too, so this looks for the kinds rather than the number) for (const wanted of ["Update run started", "Update run didn't finish", "Update run finished", "Update run stopped being followed"]) assert.ok(titles.includes(wanted), `${wanted} in ${JSON.stringify([...new Set(titles)])}`); const failedLine = events.find((e) => e.title === "Update run didn't finish")!; assert.equal(failedLine.tone, "bad"); assert.match(failedLine.detail!, /The playbook failed in Semaphore/); assert.ok((await db.select().from(auditLog)).some((a) => a.category === "update_run" && a.action === "failed")); // nothing is held in memory: a fresh pass over the table is all it takes await db.delete(updateRuns); const survivor = await startUpdateRun(input(serverId), at(20)); fake.task = { ...fake.task, status: "success", end: at(22).toISOString() }; assert.equal(await advanceUpdateRuns(at(23)), 1); assert.equal((await runRow(survivor.id)).status, "waiting_for_host"); void run; }); test("the outcome is worded plainly for each case", () => { assert.equal(summarizeOutcome(5, 0, false), "No updates are waiting now (5 updates installed); no restart is needed."); assert.equal(summarizeOutcome(1, 0, true), "No updates are waiting now (1 update installed); a restart is still needed."); assert.equal(summarizeOutcome(null, 3, false), "3 updates still waiting; no restart is needed."); assert.equal(summarizeOutcome(3, 3, false), "3 updates still waiting; no restart is needed."); assert.equal(summarizeOutcome(3, null, true), "The server reported again, but doesn't say what is waiting; a restart is still needed."); }); }); describe("the routes", () => { let app: Server; let base = ""; let who = "operator"; let serverId = 0; const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); before(async () => { await db.insert(users).values([ { oidcSub: "operator", name: "Olle", role: "operator" }, { oidcSub: "viewer", name: "Vera", role: "viewer" }, ]); const [s] = await db.insert(servers).values({ name: "zoe", apiTokenHash: hashToken("tok3"), apiTokenPrefix: "t3", lastSeenAt: T0.toISOString() }).returning(); serverId = s.id; const a = express(); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/update-runs", updateRunsRouter); app = await new Promise((resolve) => { const x = a.listen(0, "127.0.0.1", () => resolve(x)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/update-runs`; }); beforeEach(async () => { resetFake(); await db.delete(updateRuns); await db.delete(maintenanceWindows); }); after(() => app.close()); const body = (extra: object = {}) => ({ serverId, integrationId: 9, projectId: 2, templateId: 7, limit: "zoe", windowMinutes: 60, ...extra }); test("an operator starts one and everyone signed in can see it; a viewer can't start or stop one", async () => { who = "viewer"; assert.equal((await call("POST", "/", body())).status, 403); who = "operator"; const res = await call("POST", "/", body()); assert.equal(res.status, 201); const { run } = await res.json(); assert.equal(run.status, "running"); assert.equal(run.active, true); assert.equal(run.serverName, "zoe"); assert.equal(run.template, "Apply updates"); assert.equal(run.startedBy, "Olle"); who = "viewer"; const list = await (await call("GET", "/")).json(); assert.equal(list.runs.length, 1); assert.equal(list.runs[0].id, run.id); assert.equal((await (await call("GET", `/?serverId=${serverId}`)).json()).runs.length, 1); assert.equal((await (await call("GET", "/?serverId=999")).json()).runs.length, 0); assert.equal((await call("POST", `/${run.id}/cancel`)).status, 403); who = "operator"; assert.equal((await call("POST", `/${run.id}/cancel`)).status, 200); assert.equal((await call("POST", `/${run.id}/cancel`)).status, 409, "already over"); assert.equal((await call("POST", "/99999/cancel")).status, 404); }); test("what can't be done says why: a second run, a bad limit, a missing field, Semaphore refusing", async () => { assert.equal((await call("POST", "/", body())).status, 201); const again = await call("POST", "/", body()); assert.equal(again.status, 409); assert.equal((await again.json()).error, "already_running"); await db.delete(updateRuns); for (const bad of [body({ limit: "" }), body({ limit: "--extra-vars x=1" }), body({ limit: "a b" }), body({ windowMinutes: 5 }), body({ windowMinutes: 9999 }), { serverId }]) { assert.equal((await call("POST", "/", bad)).status, 400, JSON.stringify(bad)); } const limitReply = await (await call("POST", "/", body({ limit: "--extra-vars x=1" }))).json(); assert.match(limitReply.message, /host name as it is in the Semaphore inventory/); assert.equal(fake.runs.length, 1, "only the one that was fine reached Semaphore"); fake.refuseToStart = true; const refused = await call("POST", "/", body()); assert.equal(refused.status, 502); assert.match((await refused.json()).message, /Semaphore wouldn't start it/); assert.equal((await listUpdateRuns()).length, 0); }); });