Restart a server after its updates, and check that it came back

An update run can name a second Semaphore playbook that restarts the server
once the updates are in and it says a restart is due. The run keeps the
maintenance window up, follows the restart, waits for the server to report
again and checks from its uptime (or its restart flag clearing) that it did
start again. A server that only needs a restart has a Restart button on the
Updates page, and a patch plan can carry the restart playbook too.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-11-10 23:45:52 +01:00
1 parent 97df28c6dc
commit e4f3a54189
19 files changed
+10275 -87

No files matched your search

+3 -2
View File
@@ -578,10 +578,11 @@ export async function notifyStaleBackups(stale: { message: string }[]): Promise<
);
}
export async function notifyUpdateRunFinished(run: { serverName: string; templateName: string; ok: boolean; message: string }): Promise<void> {
export async function notifyUpdateRunFinished(run: { serverName: string; templateName: string; ok: boolean; message: string; kind?: "update" | "restart" }): Promise<void> {
if (!(await eventEnabled("updateRunResults"))) return;
const restart = run.kind === "restart";
await notify(
run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`,
run.ok ? `Homelab Manager — ${run.serverName} ${restart ? "restarted" : "updated"}` : `Homelab Manager — ${restart ? "Restart" : "Update"} of ${run.serverName} didn't finish`,
`"${run.templateName}" on ${run.serverName}: ${run.message}`,
{ category: "updates", severity: run.ok ? "warning" : "critical" },
);
+9 -4
View File
@@ -215,13 +215,17 @@ async function advanceRun(run: PlanRunRow, now: Date): Promise<void> {
Object.assign(next, { status: "skipped", message: "The server was removed." });
continue;
}
// A server with nothing waiting is left alone — unless it only needs a restart and the plan has a playbook for that.
let restartOnly = false;
if (plan.onlyWithUpdates) {
const updates = parsePendingUpdates(server.pendingUpdates);
if (!updates || updates.count === 0) {
if (updates && updates.count === 0 && updates.rebootRequired && plan.restartTemplateId !== null && plan.restartProjectId !== null) restartOnly = true;
else if (!updates || updates.count === 0) {
Object.assign(next, { status: "skipped", message: "No updates were waiting." });
continue;
}
}
const restartWith = plan.restartTemplateId !== null && plan.restartProjectId !== null ? { projectId: plan.restartProjectId, templateId: plan.restartTemplateId } : undefined;
const actor = `Patch plan "${plan.name}"`;
// People streaming from it (Jellyfin on this machine): its turn waits, up to two hours, rather than cutting a film off.
const watching = await watchingOnServer(server.id, now);
@@ -241,16 +245,17 @@ async function advanceRun(run: PlanRunRow, now: Date): Promise<void> {
{
serverId: server.id,
integrationId: plan.integrationId,
projectId: plan.projectId,
templateId: plan.templateId,
projectId: restartOnly ? restartWith!.projectId : plan.projectId,
templateId: restartOnly ? restartWith!.templateId : plan.templateId,
limit: plan.limitFrom === "hostname" && server.hostname ? server.hostname : server.name,
windowMinutes: plan.windowMinutes,
actor,
startAnyway: true,
...(restartOnly ? { kind: "restart" as const } : { restart: restartWith }),
},
now,
);
Object.assign(next, { status: "running", updateRunId: ur.id, message: `${waited}The update run has started.`, windowIds: await silenceDependents(plan, server.id, server.name, actor, now) });
Object.assign(next, { status: "running", updateRunId: ur.id, message: `${waited}${restartOnly ? "Nothing to update, but it needs a restart: the restart has started." : "The update run has started."}`, windowIds: await silenceDependents(plan, server.id, server.name, actor, now) });
await saveRun(run, items);
return; // one at a time: this one is followed until it is over
} catch (err) {
+3 -1
View File
@@ -87,7 +87,9 @@ export function describeAudit(category: string, action: string, detail: Record<s
const template = typeof detail.template === "string" ? detail.template : null;
switch (action) {
case "start":
return { title: "Update run started", tone: "info", kind: "change", detail: template ? `${template}, limited to ${detail.limit ?? "this server"}` : null };
return { title: detail.kind === "restart" ? "Restart started" : "Update run started", tone: "info", kind: "change", detail: template ? `${template}, limited to ${detail.limit ?? "this server"}${typeof detail.restartTemplate === "string" ? `, then a restart with ${detail.restartTemplate} if one is due` : ""}` : null };
case "restart":
return { title: "Update run is restarting the server", tone: "info", kind: "change", detail: template };
case "finished":
return { title: "Update run finished", tone: "good", kind: "change", detail: message };
case "failed":
+158 -29
View File
@@ -4,8 +4,12 @@
* for the server's agent to report again, then says how many updates are still waiting and whether a restart is due, and ends the
* maintenance window it made. Every step is written down on the run, which survives a restart of this app — a poller picks up where it was.
*
* What it does not do: it does not restart the server (the playbook does, if it is written to), and it cannot stop a run in Semaphore —
* cancelling here only stops watching and ends the window.
* A restart afterwards is a second playbook, chosen when the run is started: when the server says a restart is due once its updates are in, that playbook is
* run (the quiet is kept up for it), followed, and the server is waited for again — and it is checked that it did start again, from the uptime its agent
* reports (or, where that is not reported, from its "restart needed" flag clearing). A server that only needs a restart can be restarted on its own (kind "restart").
* Without a restart playbook the run does not restart the server (an update playbook may, if it is written to).
*
* What it cannot do: stop a run in Semaphore — cancelling here only stops watching and ends the window.
*/
import { and, desc, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
@@ -20,8 +24,13 @@ import { notifyUpdateRunFinished } from "./notify.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
export type UpdateRunRow = typeof updateRuns.$inferSelect;
export type UpdateRunStatus = "running" | "waiting_for_host" | "done" | "failed" | "cancelled";
export const ACTIVE_STATUSES: UpdateRunStatus[] = ["running", "waiting_for_host"];
export type UpdateRunStatus = "running" | "waiting_for_host" | "restarting" | "waiting_for_restart" | "done" | "failed" | "cancelled";
export const ACTIVE_STATUSES: UpdateRunStatus[] = ["running", "waiting_for_host", "restarting", "waiting_for_restart"];
export type UpdateRunKind = "update" | "restart";
/** How long the quiet is kept up for a restart, counted from when its playbook is started. */
export const RESTART_WINDOW_MINUTES = 45;
/** The uptime a report gives is worked back to a start time; a start this much before the restart began still counts (clocks and report timing are not exact). */
export const BOOT_SLACK_MS = 2 * 60_000;
/** How long a run may take from start to finish before it is given up on. */
export const RUN_LIMIT_MS = 3 * 3600_000;
@@ -71,9 +80,14 @@ export interface StartInput {
actor: string;
/** Go ahead although somebody is streaming from the machine (a person who has seen the warning, or a plan that has waited long enough). */
startAnyway?: boolean;
/** "restart": the template is a playbook that only restarts the server (nothing else is done). Default "update". */
kind?: UpdateRunKind;
/** A playbook (in the same Semaphore) that restarts the server once the updates are in, if it says a restart is due. Only for an update run. */
restart?: { projectId: number; templateId: number };
}
export async function startUpdateRun(input: StartInput, now: Date = new Date()): Promise<UpdateRunRow> {
const kind: UpdateRunKind = input.kind ?? "update";
const [server] = await db.select().from(servers).where(eq(servers.id, input.serverId)).limit(1);
if (!server) throw new UpdateRunError(404, "server_not_found", "That server doesn't exist.");
if (!server.lastSeenAt) throw new UpdateRunError(400, "never_reported", "This server has never reported, so there is nothing to wait for after the run.");
@@ -95,21 +109,28 @@ export async function startUpdateRun(input: StartInput, now: Date = new Date()):
if (!adapter) throw new UpdateRunError(400, "no_semaphore", "That Semaphore integration doesn't exist or isn't switched on.");
let templateName: string;
let restartTemplateName: string | null = null;
const restart = kind === "update" ? input.restart : undefined;
try {
const { templates } = await adapter.checkTemplates();
const template = templates.find((t) => t.projectId === input.projectId && t.id === input.templateId);
if (!template) throw new UpdateRunError(404, "template_not_found", "That template isn't in Semaphore (any more).");
templateName = template.name;
if (restart) {
const again = templates.find((t) => t.projectId === restart.projectId && t.id === restart.templateId);
if (!again) throw new UpdateRunError(404, "restart_template_not_found", "The playbook that restarts the server isn't in Semaphore (any more).");
restartTemplateName = again.name;
}
} catch (err) {
if (err instanceof UpdateRunError) throw err;
throw new UpdateRunError(502, "semaphore_unreachable", `Couldn't reach Semaphore: ${err instanceof Error ? err.message : String(err)}`);
}
// Quiet first, so nothing alerts about the server being worked on — and if the playbook can't be started the quiet is taken back.
const started = await startOrExtendWindow("server", server.id, input.windowMinutes, `Update run: ${templateName}`, input.actor, now);
const started = await startOrExtendWindow("server", server.id, input.windowMinutes, `${kind === "restart" ? "Restart" : "Update run"}: ${templateName}`, input.actor, now);
let task: SemaphoreTask;
try {
task = await adapter.runTemplate(input.projectId, input.templateId, { limit: input.limit, message: `Update run for ${server.name}, started in Homelab Manager by ${input.actor} (limited to ${input.limit})` });
task = await adapter.runTemplate(input.projectId, input.templateId, { limit: input.limit, message: `${kind === "restart" ? "Restart" : "Update run"} for ${server.name}, started in Homelab Manager by ${input.actor} (limited to ${input.limit})` });
} catch (err) {
if (started && !started.extended) await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(eq(maintenanceWindows.id, started.window.id));
invalidateAlertsCache();
@@ -136,9 +157,13 @@ export async function startUpdateRun(input: StartInput, now: Date = new Date()):
startedBy: input.actor,
startedAt: now.toISOString(),
updatesBefore: updates?.count ?? null,
kind,
restartProjectId: restart?.projectId ?? null,
restartTemplateId: restart?.templateId ?? null,
restartTemplateName,
})
.returning();
await recordAudit({ category: "update_run", action: "start", targetType: "server", targetId: server.id, detail: { run: run.id, template: templateName, limit: input.limit, task: task.id, by: input.actor } });
await recordAudit({ category: "update_run", action: "start", targetType: "server", targetId: server.id, detail: { run: run.id, kind, template: templateName, limit: input.limit, task: task.id, by: input.actor, ...(restartTemplateName ? { restartTemplate: restartTemplateName } : {}) } });
return run;
}
@@ -160,23 +185,114 @@ async function finish(run: UpdateRunRow, status: "done" | "failed" | "cancelled"
const [server] = await db.select({ name: servers.name }).from(servers).where(eq(servers.id, run.serverId)).limit(1);
await recordAudit({ category: "update_run", action: status === "done" ? "finished" : status, targetType: "server", targetId: run.serverId, detail: { run: run.id, template: run.templateName, message } });
if (status !== "cancelled") {
await notifyUpdateRunFinished({ serverName: server?.name ?? `#${run.serverId}`, templateName: run.templateName, ok: status === "done", message }).catch((err) => console.error("[updateRuns] couldn't send the result:", err));
await notifyUpdateRunFinished({ serverName: server?.name ?? `#${run.serverId}`, templateName: run.templateName, ok: status === "done", message, kind: run.kind === "restart" ? "restart" : "update" }).catch((err) => console.error("[updateRuns] couldn't send the result:", err));
}
}
const updatesWords = (n: number) => (n === 1 ? "1 update" : `${n} updates`);
/** What is left of the updates, in a clause ("2 updates still waiting (10 updates installed)"). */
function updatesClause(before: number | null, after: number | null): string {
if (after === null) return "The server reported again, but doesn't say what is waiting";
let words = after === 0 ? "No updates are waiting now" : `${updatesWords(after)} still waiting`;
if (before !== null && before > after) words += ` (${updatesWords(before - after)} installed)`;
return words;
}
/** What a finished run says about the server, from the report that came after it. */
export function summarizeOutcome(before: number | null, after: number | null, reboot: boolean): string {
const parts: string[] = [];
if (after !== null) {
parts.push(after === 0 ? "No updates are waiting now" : `${updatesWords(after)} still waiting`);
if (before !== null && before > after) parts[0] += ` (${updatesWords(before - after)} installed)`;
} else {
parts.push("The server reported again, but doesn't say what is waiting");
return `${[updatesClause(before, after), reboot ? "a restart is still needed" : "no restart is needed"].join("; ")}.`;
}
/**
* Whether the server has started since `sinceMs`, from the uptime in its last report (its start is the report's time minus its uptime): true or false, or null when it doesn't say.
* A start a little before `sinceMs` still counts (see BOOT_SLACK_MS).
*/
export function startedSince(server: { lastSeenAt: string | null; uptimeSeconds: number | null }, sinceMs: number): boolean | null {
if (!server.lastSeenAt || server.uptimeSeconds === null || server.uptimeSeconds === undefined) return null;
const seen = Date.parse(server.lastSeenAt);
if (Number.isNaN(seen)) return null;
return seen - server.uptimeSeconds * 1000 >= sinceMs - BOOT_SLACK_MS;
}
export interface RestartOutcome {
/** The server is taken to have started again. */
restarted: boolean;
/** Something said so (its uptime, or its "restart needed" flag clearing); false means it only reported again and nothing says whether it restarted. */
confirmed: boolean;
}
/** Did the restart happen? Its uptime says so where it is reported; failing that, the "restart needed" flag clearing; failing that, nothing can tell. */
export function restartOutcome(server: { lastSeenAt: string | null; uptimeSeconds: number | null }, updates: { rebootRequired: boolean } | null, sinceMs: number): RestartOutcome {
const booted = startedSince(server, sinceMs);
if (booted !== null) return { restarted: booted, confirmed: true };
if (updates) return { restarted: !updates.rebootRequired, confirmed: true };
return { restarted: true, confirmed: false };
}
/** What a finished restart says: for an update run, what is left of the updates and then the restart; for a restart on its own, only the restart. */
export function summarizeRestart(kind: UpdateRunKind, before: number | null, after: number | null, outcome: RestartOutcome, stillDue: boolean): string {
const lead = kind === "update" ? `${updatesClause(before, after)}; ` : "";
const subject = kind === "update" ? "the server" : "The server";
if (!outcome.restarted) {
return `${lead}${kind === "update" ? "the restart playbook finished" : "The playbook finished"}, but the server has not started again — check that the playbook really restarts it.`;
}
parts.push(reboot ? "a restart is still needed" : "no restart is needed");
return `${parts.join("; ")}.`;
const how = outcome.confirmed ? "" : " (its uptime isn't reported, so that couldn't be confirmed)";
return `${lead}${subject} was restarted and is back${stillDue ? ", but says another restart is due" : ""}${how}.`;
}
/** Keeps the quiet up for at least `minutes` more; a window that has ended is made again, since the server is about to restart. Returns what changed on the run, if anything. */
async function keepQuiet(run: UpdateRunRow, minutes: number, now: Date): Promise<Partial<UpdateRunRow>> {
const until = new Date(now.getTime() + minutes * 60_000).toISOString();
if (run.windowId !== null) {
const [w] = await db.select().from(maintenanceWindows).where(eq(maintenanceWindows.id, run.windowId)).limit(1);
if (w && w.endsAt > now.toISOString()) {
if (w.endsAt < until) {
await db.update(maintenanceWindows).set({ endsAt: until }).where(eq(maintenanceWindows.id, w.id));
invalidateAlertsCache();
}
return {};
}
}
const started = await startOrExtendWindow("server", run.serverId, minutes, `Update run: restart`, run.startedBy, now);
invalidateAlertsCache();
return started ? { windowId: started.window.id, windowCreated: !started.extended } : {};
}
/** The updates are in and the server says a restart is due: run the restart playbook. If it can't be started, the run ends and says the updates are in. */
async function beginRestart(run: UpdateRunRow, server: typeof servers.$inferSelect, updates: { count: number; rebootRequired: boolean } | null, now: Date): Promise<void> {
const seen = { updatesAfter: updates?.count ?? null, rebootRequired: true, hostReportedAt: server.lastSeenAt };
const adapter = await semaphoreFor(run.integrationId);
const clause = updatesClause(run.updatesBefore, updates?.count ?? null);
if (!adapter || run.restartProjectId === null || run.restartTemplateId === null) {
await finish(run, "failed", `${clause}, but the restart could not be started: Semaphore isn't available. The server still needs a restart.`, now, seen);
return;
}
const quiet = await keepQuiet(run, RESTART_WINDOW_MINUTES, now);
let task: SemaphoreTask;
try {
task = await adapter.runTemplate(run.restartProjectId, run.restartTemplateId, { limit: run.hostLimit, message: `Restart of ${server.name} after update run #${run.id}, started in Homelab Manager (limited to ${run.hostLimit})` });
} catch (err) {
await finish(run, "failed", `${clause}, but the restart could not be started: ${err instanceof Error ? err.message : String(err)}. The server still needs a restart.`, now, seen);
return;
}
await db
.update(updateRuns)
.set({ ...seen, ...quiet, status: "restarting", restartTaskId: task.id, restartTaskStatus: task.status, restartStartedAt: now.toISOString(), pollErrors: 0, message: `${clause}. The server says a restart is due — restarting it with "${run.restartTemplateName ?? "the restart playbook"}".` })
.where(eq(updateRuns.id, run.id));
await recordAudit({ category: "update_run", action: "restart", targetType: "server", targetId: run.serverId, detail: { run: run.id, template: run.restartTemplateName, task: task.id } });
}
/** A restart is over and the server has reported again: did it start again? */
async function finishRestart(run: UpdateRunRow, server: typeof servers.$inferSelect, updates: ReturnType<typeof parsePendingUpdates>, sinceMs: number, now: Date): Promise<void> {
const outcome = restartOutcome(server, updates, sinceMs);
const stillDue = updates?.rebootRequired ?? false;
await finish(run, outcome.restarted ? "done" : "failed", summarizeRestart(run.kind === "restart" ? "restart" : "update", run.updatesBefore, updates?.count ?? null, outcome, outcome.restarted && stillDue), now, {
hostReportedAt: server.lastSeenAt,
updatesAfter: updates?.count ?? null,
rebootRequired: stillDue,
restarted: outcome.restarted,
});
}
async function advance(run: UpdateRunRow, now: Date): Promise<void> {
@@ -185,11 +301,15 @@ async function advance(run: UpdateRunRow, now: Date): Promise<void> {
return;
}
if (run.status === "running") {
// The playbook is being followed: the update's, or (the second phase) the one that restarts the server.
if (run.status === "running" || run.status === "restarting") {
const restarting = run.status === "restarting";
const projectId = restarting ? run.restartProjectId : run.projectId;
const taskId = restarting ? run.restartTaskId : run.taskId;
const adapter = await semaphoreFor(run.integrationId);
let task: SemaphoreTask | null = null;
try {
task = adapter && run.taskId !== null ? await adapter.getTask(run.projectId, run.taskId) : null;
task = adapter && projectId !== null && taskId !== null ? await adapter.getTask(projectId, taskId) : null;
} catch {
task = null;
}
@@ -199,22 +319,27 @@ async function advance(run: UpdateRunRow, now: Date): Promise<void> {
else await db.update(updateRuns).set({ pollErrors: errors }).where(eq(updateRuns.id, run.id));
return;
}
const lastStatus = restarting ? run.restartTaskStatus : run.taskStatus;
const statusField = restarting ? { restartTaskStatus: task.status } : { taskStatus: task.status };
if (!FINISHED_TASK.has(task.status)) {
if (task.status !== run.taskStatus || run.pollErrors > 0) await db.update(updateRuns).set({ taskStatus: task.status, pollErrors: 0 }).where(eq(updateRuns.id, run.id));
if (task.status !== lastStatus || run.pollErrors > 0) await db.update(updateRuns).set({ ...statusField, pollErrors: 0 }).where(eq(updateRuns.id, run.id));
return;
}
if (task.status === "success") {
const ended = task.end ? new Date(task.end).toISOString() : now.toISOString();
await db.update(updateRuns).set({ status: "waiting_for_host", taskStatus: task.status, taskEndedAt: ended, pollErrors: 0, message: "The playbook finished. Waiting for the server to report again." }).where(eq(updateRuns.id, run.id));
if (restarting) await db.update(updateRuns).set({ status: "waiting_for_restart", ...statusField, restartEndedAt: ended, pollErrors: 0, message: "The restart playbook finished. Waiting for the server to come back and report." }).where(eq(updateRuns.id, run.id));
else await db.update(updateRuns).set({ status: "waiting_for_host", ...statusField, taskEndedAt: ended, pollErrors: 0, message: run.kind === "restart" ? "The restart playbook finished. Waiting for the server to come back and report." : "The playbook finished. Waiting for the server to report again." }).where(eq(updateRuns.id, run.id));
return;
}
const why = task.status === "error" ? "The playbook failed in Semaphore" : task.status === "rejected" ? "The run was rejected in Semaphore" : "The run was stopped in Semaphore";
await finish(run, "failed", `${why} (run #${run.taskId}). The server is no longer under maintenance.`, now, { taskStatus: task.status });
if (restarting) await finish(run, "failed", `The updates are in, but the restart didn't happen. ${why} (run #${taskId}). The server still needs a restart, and is no longer under maintenance.`, now, statusField);
else await finish(run, "failed", `${why} (run #${taskId}). The server is no longer under maintenance.`, now, statusField);
return;
}
// waiting_for_host: the agent has reported since the playbook ended (and, where it reports updates, those are newer than the run too).
const ended = Date.parse(run.taskEndedAt ?? run.startedAt);
// Waiting for the server: the agent has reported since the playbook ended (and, where it reports updates, those are newer than the run too).
const afterRestart = run.status === "waiting_for_restart";
const ended = Date.parse((afterRestart ? run.restartEndedAt : run.taskEndedAt) ?? (afterRestart ? run.restartStartedAt : null) ?? run.startedAt);
const [server] = await db.select().from(servers).where(eq(servers.id, run.serverId)).limit(1);
if (!server) {
await finish(run, "failed", "The server was removed.", now);
@@ -224,15 +349,19 @@ async function advance(run: UpdateRunRow, now: Date): Promise<void> {
const updates = parsePendingUpdates(server.pendingUpdates);
const updatesFresh = updates ? Date.parse(updates.reportedAt) > ended : true;
if (reported && updatesFresh) {
await finish(run, "done", summarizeOutcome(run.updatesBefore, updates?.count ?? null, updates?.rebootRequired ?? false), now, {
hostReportedAt: server.lastSeenAt,
updatesAfter: updates?.count ?? null,
rebootRequired: updates?.rebootRequired ?? false,
});
if (afterRestart) await finishRestart(run, server, updates, Date.parse(run.restartStartedAt ?? run.startedAt), now);
else if (run.kind === "restart") await finishRestart(run, server, updates, Date.parse(run.startedAt), now);
else if (run.restartTemplateId !== null && updates?.rebootRequired) await beginRestart(run, server, updates, now);
else
await finish(run, "done", summarizeOutcome(run.updatesBefore, updates?.count ?? null, updates?.rebootRequired ?? false), now, {
hostReportedAt: server.lastSeenAt,
updatesAfter: updates?.count ?? null,
rebootRequired: updates?.rebootRequired ?? false,
});
return;
}
if (now.getTime() - ended > HOST_WAIT_MS) {
await finish(run, "failed", `The playbook finished, but the server hasn't reported again in ${Math.round(HOST_WAIT_MS / 60_000)} minutes. Check that it came back.`, now);
await finish(run, "failed", `The ${afterRestart || run.kind === "restart" ? "restart playbook" : "playbook"} finished, but the server hasn't reported again in ${Math.round(HOST_WAIT_MS / 60_000)} minutes. Check that it came back.`, now);
}
}