Files
Homelab-manager/server/src/services/updateRuns.ts
T
bobbanandClaude Sonnet 5.5 0146eb8bca Jellyfin: who is watching, and don't update a machine mid-film
New Jellyfin integration (read-only, every 2 minutes): version, restart wanted,
streams and transcoding, failed scheduled tasks, item counts, failed sign-ins.
Monitoring -> Jellyfin page with a peak-streams history, three alerts, /metrics.

When Jellyfin runs on a managed server, an update run asks first if people are
streaming (409 "watching" unless startAnyway), and a patch plan waits up to two
hours for them before updating that server.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-24 22:01:18 +02:00

281 lines
15 KiB
TypeScript

/**
* An update run: apply a server's waiting updates in one go, and see it through. It puts the server under maintenance (so nothing alerts
* while it is being worked on and restarted), starts a Semaphore playbook limited to that server, follows the run until it is over, waits
* for the server's agent to report again, then says how many updates are still waiting and whether a restart is due, and ends the
* maintenance window it made. Every step is written down on the run, which survives a restart of this app — a poller picks up where it was.
*
* What it does not do: it does not restart the server (the playbook does, if it is written to), and it cannot stop a run in Semaphore —
* cancelling here only stops watching and ends the window.
*/
import { and, desc, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { maintenanceWindows, servers, updateRuns } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createSemaphoreAdapter, type SemaphoreAdapter, type SemaphoreTask } from "../integrations/semaphore/adapter.js";
import { invalidateAlertsCache } from "./alerts.js";
import { recordAudit } from "./audit.js";
import { startOrExtendWindow } from "./maintenance.js";
import { watchingOnServer, watchingWords } from "./jellyfinSync.js";
import { notifyUpdateRunFinished } from "./notify.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
export type UpdateRunRow = typeof updateRuns.$inferSelect;
export type UpdateRunStatus = "running" | "waiting_for_host" | "done" | "failed" | "cancelled";
export const ACTIVE_STATUSES: UpdateRunStatus[] = ["running", "waiting_for_host"];
/** How long a run may take from start to finish before it is given up on. */
export const RUN_LIMIT_MS = 3 * 3600_000;
/** After the playbook is over, how long the server has to report again (it may be restarting). */
export const HOST_WAIT_MS = 30 * 60_000;
/** Consecutive polls Semaphore may fail to answer before the run is given up on (at the poller's pace, about five minutes). */
export const MAX_POLL_ERRORS = 10;
export const POLL_INTERVAL_MS = 20_000;
const FINISHED_TASK = new Set<SemaphoreTask["status"]>(["success", "error", "stopped", "rejected"]);
// ─── Semaphore, replaceable for the tests ───────────────────────────────────
let adapterOverride: ((integrationId: number) => Promise<SemaphoreAdapter | null>) | null = null;
export function setSemaphoreForTests(factory: ((integrationId: number) => Promise<SemaphoreAdapter | null>) | null): void {
adapterOverride = factory;
}
async function semaphoreFor(integrationId: number): Promise<SemaphoreAdapter | null> {
if (adapterOverride) return adapterOverride(integrationId);
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "semaphore" || !loaded.integration.enabled) return null;
return createSemaphoreAdapter(loaded.config as any);
}
// ─── Starting ───────────────────────────────────────────────────────────────
export class UpdateRunError extends Error {
constructor(
readonly status: number,
readonly code: string,
message: string,
) {
super(message);
}
}
export interface StartInput {
serverId: number;
integrationId: number;
projectId: number;
templateId: number;
/** Which hosts of the inventory the playbook may touch (ansible's --limit). Never empty: an update run is always for this one server. */
limit: string;
/** How long the maintenance window lasts at most; it ends sooner when the run does. */
windowMinutes: number;
actor: string;
/** Go ahead although somebody is streaming from the machine (a person who has seen the warning, or a plan that has waited long enough). */
startAnyway?: boolean;
}
export async function startUpdateRun(input: StartInput, now: Date = new Date()): Promise<UpdateRunRow> {
const [server] = await db.select().from(servers).where(eq(servers.id, input.serverId)).limit(1);
if (!server) throw new UpdateRunError(404, "server_not_found", "That server doesn't exist.");
if (!server.lastSeenAt) throw new UpdateRunError(400, "never_reported", "This server has never reported, so there is nothing to wait for after the run.");
const [busy] = await db
.select({ id: updateRuns.id })
.from(updateRuns)
.where(and(eq(updateRuns.serverId, server.id), inArray(updateRuns.status, ACTIVE_STATUSES)))
.limit(1);
if (busy) throw new UpdateRunError(409, "already_running", `An update run for ${server.name} is already under way.`);
// Jellyfin on this machine, with people watching: the run may restart it. Said, and the person asked again, rather than cutting a film off.
if (!input.startAnyway) {
const watching = await watchingOnServer(server.id, now);
if (watching.length > 0) throw new UpdateRunError(409, "watching", `${watchingWords(watching)}. Wait until they have finished, or start it anyway.`);
}
const adapter = await semaphoreFor(input.integrationId);
if (!adapter) throw new UpdateRunError(400, "no_semaphore", "That Semaphore integration doesn't exist or isn't switched on.");
let templateName: string;
try {
const { templates } = await adapter.checkTemplates();
const template = templates.find((t) => t.projectId === input.projectId && t.id === input.templateId);
if (!template) throw new UpdateRunError(404, "template_not_found", "That template isn't in Semaphore (any more).");
templateName = template.name;
} catch (err) {
if (err instanceof UpdateRunError) throw err;
throw new UpdateRunError(502, "semaphore_unreachable", `Couldn't reach Semaphore: ${err instanceof Error ? err.message : String(err)}`);
}
// Quiet first, so nothing alerts about the server being worked on — and if the playbook can't be started the quiet is taken back.
const started = await startOrExtendWindow("server", server.id, input.windowMinutes, `Update run: ${templateName}`, input.actor, now);
let task: SemaphoreTask;
try {
task = await adapter.runTemplate(input.projectId, input.templateId, { limit: input.limit, message: `Update run for ${server.name}, started in Homelab Manager by ${input.actor} (limited to ${input.limit})` });
} catch (err) {
if (started && !started.extended) await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(eq(maintenanceWindows.id, started.window.id));
invalidateAlertsCache();
throw new UpdateRunError(502, "semaphore_failed", `Semaphore wouldn't start it: ${err instanceof Error ? err.message : String(err)}`);
}
invalidateAlertsCache();
const updates = parsePendingUpdates(server.pendingUpdates);
const [run] = await db
.insert(updateRuns)
.values({
serverId: server.id,
integrationId: input.integrationId,
projectId: input.projectId,
templateId: input.templateId,
templateName,
hostLimit: input.limit,
taskId: task.id,
taskStatus: task.status,
windowId: started?.window.id ?? null,
windowCreated: started ? !started.extended : false,
status: "running",
message: "The playbook has been started.",
startedBy: input.actor,
startedAt: now.toISOString(),
updatesBefore: updates?.count ?? null,
})
.returning();
await recordAudit({ category: "update_run", action: "start", targetType: "server", targetId: server.id, detail: { run: run.id, template: templateName, limit: input.limit, task: task.id, by: input.actor } });
return run;
}
// ─── Following ──────────────────────────────────────────────────────────────
/** Ends the maintenance window an update run made (one it only extended belongs to whoever made it, and is left to run out). */
async function endWindow(run: UpdateRunRow, now: Date): Promise<void> {
if (!run.windowCreated || run.windowId === null) return;
await db
.update(maintenanceWindows)
.set({ endsAt: now.toISOString() })
.where(and(eq(maintenanceWindows.id, run.windowId)));
invalidateAlertsCache();
}
async function finish(run: UpdateRunRow, status: "done" | "failed" | "cancelled", message: string, now: Date, extra: Partial<UpdateRunRow> = {}): Promise<void> {
await endWindow(run, now);
await db.update(updateRuns).set({ status, message, finishedAt: now.toISOString(), ...extra }).where(eq(updateRuns.id, run.id));
const [server] = await db.select({ name: servers.name }).from(servers).where(eq(servers.id, run.serverId)).limit(1);
await recordAudit({ category: "update_run", action: status === "done" ? "finished" : status, targetType: "server", targetId: run.serverId, detail: { run: run.id, template: run.templateName, message } });
if (status !== "cancelled") {
await notifyUpdateRunFinished({ serverName: server?.name ?? `#${run.serverId}`, templateName: run.templateName, ok: status === "done", message }).catch((err) => console.error("[updateRuns] couldn't send the result:", err));
}
}
const updatesWords = (n: number) => (n === 1 ? "1 update" : `${n} updates`);
/** What a finished run says about the server, from the report that came after it. */
export function summarizeOutcome(before: number | null, after: number | null, reboot: boolean): string {
const parts: string[] = [];
if (after !== null) {
parts.push(after === 0 ? "No updates are waiting now" : `${updatesWords(after)} still waiting`);
if (before !== null && before > after) parts[0] += ` (${updatesWords(before - after)} installed)`;
} else {
parts.push("The server reported again, but doesn't say what is waiting");
}
parts.push(reboot ? "a restart is still needed" : "no restart is needed");
return `${parts.join("; ")}.`;
}
async function advance(run: UpdateRunRow, now: Date): Promise<void> {
if (now.getTime() - Date.parse(run.startedAt) > RUN_LIMIT_MS) {
await finish(run, "failed", "It took longer than three hours, so it is no longer being followed. Check Semaphore.", now);
return;
}
if (run.status === "running") {
const adapter = await semaphoreFor(run.integrationId);
let task: SemaphoreTask | null = null;
try {
task = adapter && run.taskId !== null ? await adapter.getTask(run.projectId, run.taskId) : null;
} catch {
task = null;
}
if (!task) {
const errors = run.pollErrors + 1;
if (errors >= MAX_POLL_ERRORS) await finish(run, "failed", "Lost contact with Semaphore, so it is no longer being followed. Check Semaphore.", now, { pollErrors: errors });
else await db.update(updateRuns).set({ pollErrors: errors }).where(eq(updateRuns.id, run.id));
return;
}
if (!FINISHED_TASK.has(task.status)) {
if (task.status !== run.taskStatus || run.pollErrors > 0) await db.update(updateRuns).set({ taskStatus: task.status, pollErrors: 0 }).where(eq(updateRuns.id, run.id));
return;
}
if (task.status === "success") {
const ended = task.end ? new Date(task.end).toISOString() : now.toISOString();
await db.update(updateRuns).set({ status: "waiting_for_host", taskStatus: task.status, taskEndedAt: ended, pollErrors: 0, message: "The playbook finished. Waiting for the server to report again." }).where(eq(updateRuns.id, run.id));
return;
}
const why = task.status === "error" ? "The playbook failed in Semaphore" : task.status === "rejected" ? "The run was rejected in Semaphore" : "The run was stopped in Semaphore";
await finish(run, "failed", `${why} (run #${run.taskId}). The server is no longer under maintenance.`, now, { taskStatus: task.status });
return;
}
// waiting_for_host: the agent has reported since the playbook ended (and, where it reports updates, those are newer than the run too).
const ended = Date.parse(run.taskEndedAt ?? run.startedAt);
const [server] = await db.select().from(servers).where(eq(servers.id, run.serverId)).limit(1);
if (!server) {
await finish(run, "failed", "The server was removed.", now);
return;
}
const reported = server.lastSeenAt ? Date.parse(server.lastSeenAt) > ended : false;
const updates = parsePendingUpdates(server.pendingUpdates);
const updatesFresh = updates ? Date.parse(updates.reportedAt) > ended : true;
if (reported && updatesFresh) {
await finish(run, "done", summarizeOutcome(run.updatesBefore, updates?.count ?? null, updates?.rebootRequired ?? false), now, {
hostReportedAt: server.lastSeenAt,
updatesAfter: updates?.count ?? null,
rebootRequired: updates?.rebootRequired ?? false,
});
return;
}
if (now.getTime() - ended > HOST_WAIT_MS) {
await finish(run, "failed", `The playbook finished, but the server hasn't reported again in ${Math.round(HOST_WAIT_MS / 60_000)} minutes. Check that it came back.`, now);
}
}
/** One pass over every run that isn't over. Returns how many it looked at (for the tests). */
export async function advanceUpdateRuns(now: Date = new Date()): Promise<number> {
const active = await db.select().from(updateRuns).where(inArray(updateRuns.status, ACTIVE_STATUSES));
for (const run of active) {
try {
await advance(run, now);
} catch (err) {
console.error(`[updateRuns] run ${run.id}:`, err);
}
}
return active.length;
}
export async function cancelUpdateRun(id: number, actor: string, now: Date = new Date()): Promise<UpdateRunRow | null> {
const [run] = await db.select().from(updateRuns).where(eq(updateRuns.id, id)).limit(1);
if (!run) return null;
if (!ACTIVE_STATUSES.includes(run.status as UpdateRunStatus)) throw new UpdateRunError(409, "not_active", "That run is already over.");
await finish(run, "cancelled", `Stopped being followed by ${actor}. A playbook already started keeps running in Semaphore.`, now);
const [after] = await db.select().from(updateRuns).where(eq(updateRuns.id, id)).limit(1);
return after;
}
/** Recent runs, those still going first; for one server or for all. */
export async function listUpdateRuns(options: { serverId?: number; limit?: number } = {}): Promise<(UpdateRunRow & { serverName: string | null })[]> {
const rows = await db
.select({ run: updateRuns, serverName: servers.name })
.from(updateRuns)
.leftJoin(servers, eq(servers.id, updateRuns.serverId))
.where(options.serverId !== undefined ? eq(updateRuns.serverId, options.serverId) : undefined)
.orderBy(desc(updateRuns.startedAt))
.limit(options.limit ?? 30);
const out = rows.map((r) => ({ ...r.run, serverName: r.serverName }));
return [...out.filter((r) => ACTIVE_STATUSES.includes(r.status as UpdateRunStatus)), ...out.filter((r) => !ACTIVE_STATUSES.includes(r.status as UpdateRunStatus))];
}
let timer: ReturnType<typeof setInterval> | null = null;
/** Follows runs that are under way, every twenty seconds — including ones that were under way when this app last stopped. */
export function initUpdateRuns(): void {
if (timer) return;
timer = setInterval(() => void advanceUpdateRuns().catch((err) => console.error("[updateRuns]", err)), POLL_INTERVAL_MS);
}