Restart a server after its updates, and check that it came back

An update run can name a second Semaphore playbook that restarts the server
once the updates are in and it says a restart is due. The run keeps the
maintenance window up, follows the restart, waits for the server to report
again and checks from its uptime (or its restart flag clearing) that it did
start again. A server that only needs a restart has a Restart button on the
Updates page, and a patch plan can carry the restart playbook too.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-11-10 23:45:52 +01:00
1 parent 97df28c6dc
commit e4f3a54189
19 files changed
+10275 -87

No files matched your search

+16 -1
View File
@@ -1003,7 +1003,8 @@ export const incidentNotes = sqliteTable(
// ─── Update runs: apply a server's updates and see it through ──────────────
// A playbook run in Semaphore for one server, wrapped in a maintenance window and followed until the server reports again — see
// services/updateRuns.ts. status: running | waiting_for_host | done | failed | cancelled.
// services/updateRuns.ts. status: running | waiting_for_host | restarting | waiting_for_restart | done | failed | cancelled. kind: "update" (apply updates, and restart
// afterwards if a restart template is set and the server says one is due) or "restart" (the template is a restart playbook and nothing else is done).
export const updateRuns = sqliteTable(
"update_runs",
{
@@ -1031,6 +1032,16 @@ export const updateRuns = sqliteTable(
updatesAfter: integer("updates_after"),
rebootRequired: integer("reboot_required", { mode: "boolean" }).notNull().default(false),
pollErrors: integer("poll_errors").notNull().default(0),
kind: text("kind").notNull().default("update"),
// The playbook that restarts the server afterwards (same Semaphore integration), and how that went.
restartProjectId: integer("restart_project_id"),
restartTemplateId: integer("restart_template_id"),
restartTemplateName: text("restart_template_name"),
restartTaskId: integer("restart_task_id"),
restartTaskStatus: text("restart_task_status"),
restartStartedAt: text("restart_started_at"), // ISO
restartEndedAt: text("restart_ended_at"), // ISO — when the restart playbook finished
restarted: integer("restarted", { mode: "boolean" }).notNull().default(false), // the server was seen to have started again (from its uptime, or its restart flag clearing)
},
(t) => [index("update_runs_server_idx").on(t.serverId, t.startedAt), index("update_runs_status_idx").on(t.status)],
);
@@ -1052,6 +1063,10 @@ export const patchPlans = sqliteTable("patch_plans", {
projectId: integer("project_id").notNull(),
templateId: integer("template_id").notNull(),
templateName: text("template_name").notNull(),
// A playbook that restarts a server that says a restart is due once its updates are in (and one that only needs a restart is restarted instead of left alone); null = none.
restartProjectId: integer("restart_project_id"),
restartTemplateId: integer("restart_template_id"),
restartTemplateName: text("restart_template_name"),
limitFrom: text("limit_from").notNull().default("name"), // what ansible's --limit is for a server: its name here, or its host name
stopOnFailure: integer("stop_on_failure", { mode: "boolean" }).notNull().default(true),
onlyWithUpdates: integer("only_with_updates", { mode: "boolean" }).notNull().default(true), // leave out a server that has nothing waiting