/** * An update run: apply a server's waiting updates in one go, and see it through. It puts the server under maintenance (so nothing alerts * while it is being worked on and restarted), starts a Semaphore playbook limited to that server, follows the run until it is over, waits * for the server's agent to report again, then says how many updates are still waiting and whether a restart is due, and ends the * maintenance window it made. Every step is written down on the run, which survives a restart of this app — a poller picks up where it was. * * What it does not do: it does not restart the server (the playbook does, if it is written to), and it cannot stop a run in Semaphore — * cancelling here only stops watching and ends the window. */ import { and, desc, eq, inArray } from "drizzle-orm"; import { db } from "../db/client.js"; import { maintenanceWindows, servers, updateRuns } from "../db/schema.js"; import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; import { createSemaphoreAdapter, type SemaphoreAdapter, type SemaphoreTask } from "../integrations/semaphore/adapter.js"; import { invalidateAlertsCache } from "./alerts.js"; import { recordAudit } from "./audit.js"; import { startOrExtendWindow } from "./maintenance.js"; import { watchingOnServer, watchingWords } from "./jellyfinSync.js"; import { notifyUpdateRunFinished } from "./notify.js"; import { parsePendingUpdates } from "./pendingUpdates.js"; export type UpdateRunRow = typeof updateRuns.$inferSelect; export type UpdateRunStatus = "running" | "waiting_for_host" | "done" | "failed" | "cancelled"; export const ACTIVE_STATUSES: UpdateRunStatus[] = ["running", "waiting_for_host"]; /** How long a run may take from start to finish before it is given up on. */ export const RUN_LIMIT_MS = 3 * 3600_000; /** After the playbook is over, how long the server has to report again (it may be restarting). */ export const HOST_WAIT_MS = 30 * 60_000; /** Consecutive polls Semaphore may fail to answer before the run is given up on (at the poller's pace, about five minutes). */ export const MAX_POLL_ERRORS = 10; export const POLL_INTERVAL_MS = 20_000; const FINISHED_TASK = new Set(["success", "error", "stopped", "rejected"]); // ─── Semaphore, replaceable for the tests ─────────────────────────────────── let adapterOverride: ((integrationId: number) => Promise) | null = null; export function setSemaphoreForTests(factory: ((integrationId: number) => Promise) | null): void { adapterOverride = factory; } async function semaphoreFor(integrationId: number): Promise { if (adapterOverride) return adapterOverride(integrationId); const loaded = await loadIntegrationConfig(integrationId); if (!loaded || loaded.integration.type !== "semaphore" || !loaded.integration.enabled) return null; return createSemaphoreAdapter(loaded.config as any); } // ─── Starting ─────────────────────────────────────────────────────────────── export class UpdateRunError extends Error { constructor( readonly status: number, readonly code: string, message: string, ) { super(message); } } export interface StartInput { serverId: number; integrationId: number; projectId: number; templateId: number; /** Which hosts of the inventory the playbook may touch (ansible's --limit). Never empty: an update run is always for this one server. */ limit: string; /** How long the maintenance window lasts at most; it ends sooner when the run does. */ windowMinutes: number; actor: string; /** Go ahead although somebody is streaming from the machine (a person who has seen the warning, or a plan that has waited long enough). */ startAnyway?: boolean; } export async function startUpdateRun(input: StartInput, now: Date = new Date()): Promise { const [server] = await db.select().from(servers).where(eq(servers.id, input.serverId)).limit(1); if (!server) throw new UpdateRunError(404, "server_not_found", "That server doesn't exist."); if (!server.lastSeenAt) throw new UpdateRunError(400, "never_reported", "This server has never reported, so there is nothing to wait for after the run."); const [busy] = await db .select({ id: updateRuns.id }) .from(updateRuns) .where(and(eq(updateRuns.serverId, server.id), inArray(updateRuns.status, ACTIVE_STATUSES))) .limit(1); if (busy) throw new UpdateRunError(409, "already_running", `An update run for ${server.name} is already under way.`); // Jellyfin on this machine, with people watching: the run may restart it. Said, and the person asked again, rather than cutting a film off. if (!input.startAnyway) { const watching = await watchingOnServer(server.id, now); if (watching.length > 0) throw new UpdateRunError(409, "watching", `${watchingWords(watching)}. Wait until they have finished, or start it anyway.`); } const adapter = await semaphoreFor(input.integrationId); if (!adapter) throw new UpdateRunError(400, "no_semaphore", "That Semaphore integration doesn't exist or isn't switched on."); let templateName: string; try { const { templates } = await adapter.checkTemplates(); const template = templates.find((t) => t.projectId === input.projectId && t.id === input.templateId); if (!template) throw new UpdateRunError(404, "template_not_found", "That template isn't in Semaphore (any more)."); templateName = template.name; } catch (err) { if (err instanceof UpdateRunError) throw err; throw new UpdateRunError(502, "semaphore_unreachable", `Couldn't reach Semaphore: ${err instanceof Error ? err.message : String(err)}`); } // Quiet first, so nothing alerts about the server being worked on — and if the playbook can't be started the quiet is taken back. const started = await startOrExtendWindow("server", server.id, input.windowMinutes, `Update run: ${templateName}`, input.actor, now); let task: SemaphoreTask; try { task = await adapter.runTemplate(input.projectId, input.templateId, { limit: input.limit, message: `Update run for ${server.name}, started in Homelab Manager by ${input.actor} (limited to ${input.limit})` }); } catch (err) { if (started && !started.extended) await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(eq(maintenanceWindows.id, started.window.id)); invalidateAlertsCache(); throw new UpdateRunError(502, "semaphore_failed", `Semaphore wouldn't start it: ${err instanceof Error ? err.message : String(err)}`); } invalidateAlertsCache(); const updates = parsePendingUpdates(server.pendingUpdates); const [run] = await db .insert(updateRuns) .values({ serverId: server.id, integrationId: input.integrationId, projectId: input.projectId, templateId: input.templateId, templateName, hostLimit: input.limit, taskId: task.id, taskStatus: task.status, windowId: started?.window.id ?? null, windowCreated: started ? !started.extended : false, status: "running", message: "The playbook has been started.", startedBy: input.actor, startedAt: now.toISOString(), updatesBefore: updates?.count ?? null, }) .returning(); await recordAudit({ category: "update_run", action: "start", targetType: "server", targetId: server.id, detail: { run: run.id, template: templateName, limit: input.limit, task: task.id, by: input.actor } }); return run; } // ─── Following ────────────────────────────────────────────────────────────── /** Ends the maintenance window an update run made (one it only extended belongs to whoever made it, and is left to run out). */ async function endWindow(run: UpdateRunRow, now: Date): Promise { if (!run.windowCreated || run.windowId === null) return; await db .update(maintenanceWindows) .set({ endsAt: now.toISOString() }) .where(and(eq(maintenanceWindows.id, run.windowId))); invalidateAlertsCache(); } async function finish(run: UpdateRunRow, status: "done" | "failed" | "cancelled", message: string, now: Date, extra: Partial = {}): Promise { await endWindow(run, now); await db.update(updateRuns).set({ status, message, finishedAt: now.toISOString(), ...extra }).where(eq(updateRuns.id, run.id)); const [server] = await db.select({ name: servers.name }).from(servers).where(eq(servers.id, run.serverId)).limit(1); await recordAudit({ category: "update_run", action: status === "done" ? "finished" : status, targetType: "server", targetId: run.serverId, detail: { run: run.id, template: run.templateName, message } }); if (status !== "cancelled") { await notifyUpdateRunFinished({ serverName: server?.name ?? `#${run.serverId}`, templateName: run.templateName, ok: status === "done", message }).catch((err) => console.error("[updateRuns] couldn't send the result:", err)); } } const updatesWords = (n: number) => (n === 1 ? "1 update" : `${n} updates`); /** What a finished run says about the server, from the report that came after it. */ export function summarizeOutcome(before: number | null, after: number | null, reboot: boolean): string { const parts: string[] = []; if (after !== null) { parts.push(after === 0 ? "No updates are waiting now" : `${updatesWords(after)} still waiting`); if (before !== null && before > after) parts[0] += ` (${updatesWords(before - after)} installed)`; } else { parts.push("The server reported again, but doesn't say what is waiting"); } parts.push(reboot ? "a restart is still needed" : "no restart is needed"); return `${parts.join("; ")}.`; } async function advance(run: UpdateRunRow, now: Date): Promise { if (now.getTime() - Date.parse(run.startedAt) > RUN_LIMIT_MS) { await finish(run, "failed", "It took longer than three hours, so it is no longer being followed. Check Semaphore.", now); return; } if (run.status === "running") { const adapter = await semaphoreFor(run.integrationId); let task: SemaphoreTask | null = null; try { task = adapter && run.taskId !== null ? await adapter.getTask(run.projectId, run.taskId) : null; } catch { task = null; } if (!task) { const errors = run.pollErrors + 1; if (errors >= MAX_POLL_ERRORS) await finish(run, "failed", "Lost contact with Semaphore, so it is no longer being followed. Check Semaphore.", now, { pollErrors: errors }); else await db.update(updateRuns).set({ pollErrors: errors }).where(eq(updateRuns.id, run.id)); return; } if (!FINISHED_TASK.has(task.status)) { if (task.status !== run.taskStatus || run.pollErrors > 0) await db.update(updateRuns).set({ taskStatus: task.status, pollErrors: 0 }).where(eq(updateRuns.id, run.id)); return; } if (task.status === "success") { const ended = task.end ? new Date(task.end).toISOString() : now.toISOString(); await db.update(updateRuns).set({ status: "waiting_for_host", taskStatus: task.status, taskEndedAt: ended, pollErrors: 0, message: "The playbook finished. Waiting for the server to report again." }).where(eq(updateRuns.id, run.id)); return; } const why = task.status === "error" ? "The playbook failed in Semaphore" : task.status === "rejected" ? "The run was rejected in Semaphore" : "The run was stopped in Semaphore"; await finish(run, "failed", `${why} (run #${run.taskId}). The server is no longer under maintenance.`, now, { taskStatus: task.status }); return; } // waiting_for_host: the agent has reported since the playbook ended (and, where it reports updates, those are newer than the run too). const ended = Date.parse(run.taskEndedAt ?? run.startedAt); const [server] = await db.select().from(servers).where(eq(servers.id, run.serverId)).limit(1); if (!server) { await finish(run, "failed", "The server was removed.", now); return; } const reported = server.lastSeenAt ? Date.parse(server.lastSeenAt) > ended : false; const updates = parsePendingUpdates(server.pendingUpdates); const updatesFresh = updates ? Date.parse(updates.reportedAt) > ended : true; if (reported && updatesFresh) { await finish(run, "done", summarizeOutcome(run.updatesBefore, updates?.count ?? null, updates?.rebootRequired ?? false), now, { hostReportedAt: server.lastSeenAt, updatesAfter: updates?.count ?? null, rebootRequired: updates?.rebootRequired ?? false, }); return; } if (now.getTime() - ended > HOST_WAIT_MS) { await finish(run, "failed", `The playbook finished, but the server hasn't reported again in ${Math.round(HOST_WAIT_MS / 60_000)} minutes. Check that it came back.`, now); } } /** One pass over every run that isn't over. Returns how many it looked at (for the tests). */ export async function advanceUpdateRuns(now: Date = new Date()): Promise { const active = await db.select().from(updateRuns).where(inArray(updateRuns.status, ACTIVE_STATUSES)); for (const run of active) { try { await advance(run, now); } catch (err) { console.error(`[updateRuns] run ${run.id}:`, err); } } return active.length; } export async function cancelUpdateRun(id: number, actor: string, now: Date = new Date()): Promise { const [run] = await db.select().from(updateRuns).where(eq(updateRuns.id, id)).limit(1); if (!run) return null; if (!ACTIVE_STATUSES.includes(run.status as UpdateRunStatus)) throw new UpdateRunError(409, "not_active", "That run is already over."); await finish(run, "cancelled", `Stopped being followed by ${actor}. A playbook already started keeps running in Semaphore.`, now); const [after] = await db.select().from(updateRuns).where(eq(updateRuns.id, id)).limit(1); return after; } /** Recent runs, those still going first; for one server or for all. */ export async function listUpdateRuns(options: { serverId?: number; limit?: number } = {}): Promise<(UpdateRunRow & { serverName: string | null })[]> { const rows = await db .select({ run: updateRuns, serverName: servers.name }) .from(updateRuns) .leftJoin(servers, eq(servers.id, updateRuns.serverId)) .where(options.serverId !== undefined ? eq(updateRuns.serverId, options.serverId) : undefined) .orderBy(desc(updateRuns.startedAt)) .limit(options.limit ?? 30); const out = rows.map((r) => ({ ...r.run, serverName: r.serverName })); return [...out.filter((r) => ACTIVE_STATUSES.includes(r.status as UpdateRunStatus)), ...out.filter((r) => !ACTIVE_STATUSES.includes(r.status as UpdateRunStatus))]; } let timer: ReturnType | null = null; /** Follows runs that are under way, every twenty seconds — including ones that were under way when this app last stopped. */ export function initUpdateRuns(): void { if (timer) return; timer = setInterval(() => void advanceUpdateRuns().catch((err) => console.error("[updateRuns]", err)), POLL_INTERVAL_MS); }