Restart a server after its updates, and check that it came back
An update run can name a second Semaphore playbook that restarts the server once the updates are in and it says a restart is due. The run keeps the maintenance window up, follows the restart, waits for the server to report again and checks from its uptime (or its restart flag clearing) that it did start again. A server that only needs a restart has a Restart button on the Updates page, and a patch plan can carry the restart playbook too. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
97df28c6dc
commit
e4f3a54189
19 files changed
+10275
-87
No files matched your search
@@ -0,0 +1,12 @@
|
||||
ALTER TABLE `patch_plans` ADD `restart_project_id` integer;--> statement-breakpoint
|
||||
ALTER TABLE `patch_plans` ADD `restart_template_id` integer;--> statement-breakpoint
|
||||
ALTER TABLE `patch_plans` ADD `restart_template_name` text;--> statement-breakpoint
|
||||
ALTER TABLE `update_runs` ADD `kind` text DEFAULT 'update' NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE `update_runs` ADD `restart_project_id` integer;--> statement-breakpoint
|
||||
ALTER TABLE `update_runs` ADD `restart_template_id` integer;--> statement-breakpoint
|
||||
ALTER TABLE `update_runs` ADD `restart_template_name` text;--> statement-breakpoint
|
||||
ALTER TABLE `update_runs` ADD `restart_task_id` integer;--> statement-breakpoint
|
||||
ALTER TABLE `update_runs` ADD `restart_task_status` text;--> statement-breakpoint
|
||||
ALTER TABLE `update_runs` ADD `restart_started_at` text;--> statement-breakpoint
|
||||
ALTER TABLE `update_runs` ADD `restart_ended_at` text;--> statement-breakpoint
|
||||
ALTER TABLE `update_runs` ADD `restarted` integer DEFAULT false NOT NULL;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -652,6 +652,13 @@
|
||||
"when": 1794340893429,
|
||||
"tag": "0092_parallel_the_initiative",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 93,
|
||||
"version": "6",
|
||||
"when": 1794349689431,
|
||||
"tag": "0093_reboot_after_updates",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
+16
-1
@@ -1003,7 +1003,8 @@ export const incidentNotes = sqliteTable(
|
||||
// ─── Update runs: apply a server's updates and see it through ──────────────
|
||||
|
||||
// A playbook run in Semaphore for one server, wrapped in a maintenance window and followed until the server reports again — see
|
||||
// services/updateRuns.ts. status: running | waiting_for_host | done | failed | cancelled.
|
||||
// services/updateRuns.ts. status: running | waiting_for_host | restarting | waiting_for_restart | done | failed | cancelled. kind: "update" (apply updates, and restart
|
||||
// afterwards if a restart template is set and the server says one is due) or "restart" (the template is a restart playbook and nothing else is done).
|
||||
export const updateRuns = sqliteTable(
|
||||
"update_runs",
|
||||
{
|
||||
@@ -1031,6 +1032,16 @@ export const updateRuns = sqliteTable(
|
||||
updatesAfter: integer("updates_after"),
|
||||
rebootRequired: integer("reboot_required", { mode: "boolean" }).notNull().default(false),
|
||||
pollErrors: integer("poll_errors").notNull().default(0),
|
||||
kind: text("kind").notNull().default("update"),
|
||||
// The playbook that restarts the server afterwards (same Semaphore integration), and how that went.
|
||||
restartProjectId: integer("restart_project_id"),
|
||||
restartTemplateId: integer("restart_template_id"),
|
||||
restartTemplateName: text("restart_template_name"),
|
||||
restartTaskId: integer("restart_task_id"),
|
||||
restartTaskStatus: text("restart_task_status"),
|
||||
restartStartedAt: text("restart_started_at"), // ISO
|
||||
restartEndedAt: text("restart_ended_at"), // ISO — when the restart playbook finished
|
||||
restarted: integer("restarted", { mode: "boolean" }).notNull().default(false), // the server was seen to have started again (from its uptime, or its restart flag clearing)
|
||||
},
|
||||
(t) => [index("update_runs_server_idx").on(t.serverId, t.startedAt), index("update_runs_status_idx").on(t.status)],
|
||||
);
|
||||
@@ -1052,6 +1063,10 @@ export const patchPlans = sqliteTable("patch_plans", {
|
||||
projectId: integer("project_id").notNull(),
|
||||
templateId: integer("template_id").notNull(),
|
||||
templateName: text("template_name").notNull(),
|
||||
// A playbook that restarts a server that says a restart is due once its updates are in (and one that only needs a restart is restarted instead of left alone); null = none.
|
||||
restartProjectId: integer("restart_project_id"),
|
||||
restartTemplateId: integer("restart_template_id"),
|
||||
restartTemplateName: text("restart_template_name"),
|
||||
limitFrom: text("limit_from").notNull().default("name"), // what ansible's --limit is for a server: its name here, or its host name
|
||||
stopOnFailure: integer("stop_on_failure", { mode: "boolean" }).notNull().default(true),
|
||||
onlyWithUpdates: integer("only_with_updates", { mode: "boolean" }).notNull().default(true), // leave out a server that has nothing waiting
|
||||
|
||||
@@ -36,7 +36,12 @@ const planSchema = z
|
||||
onlyWithUpdates: z.boolean().default(true),
|
||||
silenceDependents: z.boolean().default(true),
|
||||
windowMinutes: z.number().int().min(15).max(480).default(90),
|
||||
// A playbook that restarts a server that says a restart is due after its updates (all three or none).
|
||||
restartProjectId: z.number().int().positive().nullable().default(null),
|
||||
restartTemplateId: z.number().int().positive().nullable().default(null),
|
||||
restartTemplateName: z.string().trim().min(1).max(200).nullable().default(null),
|
||||
})
|
||||
.refine((d) => (d.restartProjectId === null) === (d.restartTemplateId === null) && (d.restartTemplateId === null) === (d.restartTemplateName === null), { message: "The restart playbook needs its project, its template and its name — or none of them.", path: ["restartTemplateId"] })
|
||||
.refine((d) => isValidTimeZone(d.timezone), { message: "That isn't a time zone the system knows (use a name like Europe/Stockholm).", path: ["timezone"] })
|
||||
.refine((d) => d.tags.length > 0 || d.serverIds.length > 0, { message: "Choose which servers: give a tag, or name at least one server.", path: ["tags"] });
|
||||
|
||||
@@ -58,6 +63,9 @@ function valuesOf(d: z.infer<typeof planSchema>) {
|
||||
onlyWithUpdates: d.onlyWithUpdates,
|
||||
silenceDependents: d.silenceDependents,
|
||||
windowMinutes: d.windowMinutes,
|
||||
restartProjectId: d.restartProjectId,
|
||||
restartTemplateId: d.restartTemplateId,
|
||||
restartTemplateName: d.restartTemplateName,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -96,6 +104,9 @@ async function shapePlan(p: PlanRow, all: { id: number; name: string; tags: stri
|
||||
onlyWithUpdates: p.onlyWithUpdates,
|
||||
silenceDependents: p.silenceDependents,
|
||||
windowMinutes: p.windowMinutes,
|
||||
restartProjectId: p.restartProjectId,
|
||||
restartTemplateId: p.restartTemplateId,
|
||||
restartTemplateName: p.restartTemplateName,
|
||||
createdBy: p.createdBy,
|
||||
servers: covered,
|
||||
next: next.map((o) => o.start.toISOString()),
|
||||
|
||||
@@ -20,6 +20,10 @@ const startSchema = z.object({
|
||||
windowMinutes: z.number().int().min(15).max(480).default(90),
|
||||
// Somebody is streaming from the machine and the person has seen the warning.
|
||||
startAnyway: z.boolean().optional(),
|
||||
// "restart": the template is a playbook that only restarts the server.
|
||||
kind: z.enum(["update", "restart"]).default("update"),
|
||||
// An update run's playbook that restarts the server afterwards if it says one is due (in the same Semaphore).
|
||||
restart: z.object({ projectId: z.number().int().positive(), templateId: z.number().int().positive() }).optional(),
|
||||
});
|
||||
|
||||
const shape = (r: Awaited<ReturnType<typeof listUpdateRuns>>[number]) => ({
|
||||
@@ -41,6 +45,12 @@ const shape = (r: Awaited<ReturnType<typeof listUpdateRuns>>[number]) => ({
|
||||
updatesBefore: r.updatesBefore,
|
||||
updatesAfter: r.updatesAfter,
|
||||
rebootRequired: r.rebootRequired,
|
||||
kind: r.kind,
|
||||
restartTemplate: r.restartTemplateName,
|
||||
restartTaskId: r.restartTaskId,
|
||||
restartTaskStatus: r.restartTaskStatus,
|
||||
restartedAt: r.restartStartedAt,
|
||||
restarted: r.restarted,
|
||||
});
|
||||
|
||||
updateRunsRouter.get("/", asyncHandler(async (req, res) => {
|
||||
|
||||
@@ -578,10 +578,11 @@ export async function notifyStaleBackups(stale: { message: string }[]): Promise<
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyUpdateRunFinished(run: { serverName: string; templateName: string; ok: boolean; message: string }): Promise<void> {
|
||||
export async function notifyUpdateRunFinished(run: { serverName: string; templateName: string; ok: boolean; message: string; kind?: "update" | "restart" }): Promise<void> {
|
||||
if (!(await eventEnabled("updateRunResults"))) return;
|
||||
const restart = run.kind === "restart";
|
||||
await notify(
|
||||
run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`,
|
||||
run.ok ? `Homelab Manager — ${run.serverName} ${restart ? "restarted" : "updated"}` : `Homelab Manager — ${restart ? "Restart" : "Update"} of ${run.serverName} didn't finish`,
|
||||
`"${run.templateName}" on ${run.serverName}: ${run.message}`,
|
||||
{ category: "updates", severity: run.ok ? "warning" : "critical" },
|
||||
);
|
||||
|
||||
@@ -215,13 +215,17 @@ async function advanceRun(run: PlanRunRow, now: Date): Promise<void> {
|
||||
Object.assign(next, { status: "skipped", message: "The server was removed." });
|
||||
continue;
|
||||
}
|
||||
// A server with nothing waiting is left alone — unless it only needs a restart and the plan has a playbook for that.
|
||||
let restartOnly = false;
|
||||
if (plan.onlyWithUpdates) {
|
||||
const updates = parsePendingUpdates(server.pendingUpdates);
|
||||
if (!updates || updates.count === 0) {
|
||||
if (updates && updates.count === 0 && updates.rebootRequired && plan.restartTemplateId !== null && plan.restartProjectId !== null) restartOnly = true;
|
||||
else if (!updates || updates.count === 0) {
|
||||
Object.assign(next, { status: "skipped", message: "No updates were waiting." });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const restartWith = plan.restartTemplateId !== null && plan.restartProjectId !== null ? { projectId: plan.restartProjectId, templateId: plan.restartTemplateId } : undefined;
|
||||
const actor = `Patch plan "${plan.name}"`;
|
||||
// People streaming from it (Jellyfin on this machine): its turn waits, up to two hours, rather than cutting a film off.
|
||||
const watching = await watchingOnServer(server.id, now);
|
||||
@@ -241,16 +245,17 @@ async function advanceRun(run: PlanRunRow, now: Date): Promise<void> {
|
||||
{
|
||||
serverId: server.id,
|
||||
integrationId: plan.integrationId,
|
||||
projectId: plan.projectId,
|
||||
templateId: plan.templateId,
|
||||
projectId: restartOnly ? restartWith!.projectId : plan.projectId,
|
||||
templateId: restartOnly ? restartWith!.templateId : plan.templateId,
|
||||
limit: plan.limitFrom === "hostname" && server.hostname ? server.hostname : server.name,
|
||||
windowMinutes: plan.windowMinutes,
|
||||
actor,
|
||||
startAnyway: true,
|
||||
...(restartOnly ? { kind: "restart" as const } : { restart: restartWith }),
|
||||
},
|
||||
now,
|
||||
);
|
||||
Object.assign(next, { status: "running", updateRunId: ur.id, message: `${waited}The update run has started.`, windowIds: await silenceDependents(plan, server.id, server.name, actor, now) });
|
||||
Object.assign(next, { status: "running", updateRunId: ur.id, message: `${waited}${restartOnly ? "Nothing to update, but it needs a restart: the restart has started." : "The update run has started."}`, windowIds: await silenceDependents(plan, server.id, server.name, actor, now) });
|
||||
await saveRun(run, items);
|
||||
return; // one at a time: this one is followed until it is over
|
||||
} catch (err) {
|
||||
|
||||
@@ -87,7 +87,9 @@ export function describeAudit(category: string, action: string, detail: Record<s
|
||||
const template = typeof detail.template === "string" ? detail.template : null;
|
||||
switch (action) {
|
||||
case "start":
|
||||
return { title: "Update run started", tone: "info", kind: "change", detail: template ? `${template}, limited to ${detail.limit ?? "this server"}` : null };
|
||||
return { title: detail.kind === "restart" ? "Restart started" : "Update run started", tone: "info", kind: "change", detail: template ? `${template}, limited to ${detail.limit ?? "this server"}${typeof detail.restartTemplate === "string" ? `, then a restart with ${detail.restartTemplate} if one is due` : ""}` : null };
|
||||
case "restart":
|
||||
return { title: "Update run is restarting the server", tone: "info", kind: "change", detail: template };
|
||||
case "finished":
|
||||
return { title: "Update run finished", tone: "good", kind: "change", detail: message };
|
||||
case "failed":
|
||||
|
||||
@@ -4,8 +4,12 @@
|
||||
* for the server's agent to report again, then says how many updates are still waiting and whether a restart is due, and ends the
|
||||
* maintenance window it made. Every step is written down on the run, which survives a restart of this app — a poller picks up where it was.
|
||||
*
|
||||
* What it does not do: it does not restart the server (the playbook does, if it is written to), and it cannot stop a run in Semaphore —
|
||||
* cancelling here only stops watching and ends the window.
|
||||
* A restart afterwards is a second playbook, chosen when the run is started: when the server says a restart is due once its updates are in, that playbook is
|
||||
* run (the quiet is kept up for it), followed, and the server is waited for again — and it is checked that it did start again, from the uptime its agent
|
||||
* reports (or, where that is not reported, from its "restart needed" flag clearing). A server that only needs a restart can be restarted on its own (kind "restart").
|
||||
* Without a restart playbook the run does not restart the server (an update playbook may, if it is written to).
|
||||
*
|
||||
* What it cannot do: stop a run in Semaphore — cancelling here only stops watching and ends the window.
|
||||
*/
|
||||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
@@ -20,8 +24,13 @@ import { notifyUpdateRunFinished } from "./notify.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
|
||||
export type UpdateRunRow = typeof updateRuns.$inferSelect;
|
||||
export type UpdateRunStatus = "running" | "waiting_for_host" | "done" | "failed" | "cancelled";
|
||||
export const ACTIVE_STATUSES: UpdateRunStatus[] = ["running", "waiting_for_host"];
|
||||
export type UpdateRunStatus = "running" | "waiting_for_host" | "restarting" | "waiting_for_restart" | "done" | "failed" | "cancelled";
|
||||
export const ACTIVE_STATUSES: UpdateRunStatus[] = ["running", "waiting_for_host", "restarting", "waiting_for_restart"];
|
||||
export type UpdateRunKind = "update" | "restart";
|
||||
/** How long the quiet is kept up for a restart, counted from when its playbook is started. */
|
||||
export const RESTART_WINDOW_MINUTES = 45;
|
||||
/** The uptime a report gives is worked back to a start time; a start this much before the restart began still counts (clocks and report timing are not exact). */
|
||||
export const BOOT_SLACK_MS = 2 * 60_000;
|
||||
|
||||
/** How long a run may take from start to finish before it is given up on. */
|
||||
export const RUN_LIMIT_MS = 3 * 3600_000;
|
||||
@@ -71,9 +80,14 @@ export interface StartInput {
|
||||
actor: string;
|
||||
/** Go ahead although somebody is streaming from the machine (a person who has seen the warning, or a plan that has waited long enough). */
|
||||
startAnyway?: boolean;
|
||||
/** "restart": the template is a playbook that only restarts the server (nothing else is done). Default "update". */
|
||||
kind?: UpdateRunKind;
|
||||
/** A playbook (in the same Semaphore) that restarts the server once the updates are in, if it says a restart is due. Only for an update run. */
|
||||
restart?: { projectId: number; templateId: number };
|
||||
}
|
||||
|
||||
export async function startUpdateRun(input: StartInput, now: Date = new Date()): Promise<UpdateRunRow> {
|
||||
const kind: UpdateRunKind = input.kind ?? "update";
|
||||
const [server] = await db.select().from(servers).where(eq(servers.id, input.serverId)).limit(1);
|
||||
if (!server) throw new UpdateRunError(404, "server_not_found", "That server doesn't exist.");
|
||||
if (!server.lastSeenAt) throw new UpdateRunError(400, "never_reported", "This server has never reported, so there is nothing to wait for after the run.");
|
||||
@@ -95,21 +109,28 @@ export async function startUpdateRun(input: StartInput, now: Date = new Date()):
|
||||
if (!adapter) throw new UpdateRunError(400, "no_semaphore", "That Semaphore integration doesn't exist or isn't switched on.");
|
||||
|
||||
let templateName: string;
|
||||
let restartTemplateName: string | null = null;
|
||||
const restart = kind === "update" ? input.restart : undefined;
|
||||
try {
|
||||
const { templates } = await adapter.checkTemplates();
|
||||
const template = templates.find((t) => t.projectId === input.projectId && t.id === input.templateId);
|
||||
if (!template) throw new UpdateRunError(404, "template_not_found", "That template isn't in Semaphore (any more).");
|
||||
templateName = template.name;
|
||||
if (restart) {
|
||||
const again = templates.find((t) => t.projectId === restart.projectId && t.id === restart.templateId);
|
||||
if (!again) throw new UpdateRunError(404, "restart_template_not_found", "The playbook that restarts the server isn't in Semaphore (any more).");
|
||||
restartTemplateName = again.name;
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof UpdateRunError) throw err;
|
||||
throw new UpdateRunError(502, "semaphore_unreachable", `Couldn't reach Semaphore: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
// Quiet first, so nothing alerts about the server being worked on — and if the playbook can't be started the quiet is taken back.
|
||||
const started = await startOrExtendWindow("server", server.id, input.windowMinutes, `Update run: ${templateName}`, input.actor, now);
|
||||
const started = await startOrExtendWindow("server", server.id, input.windowMinutes, `${kind === "restart" ? "Restart" : "Update run"}: ${templateName}`, input.actor, now);
|
||||
let task: SemaphoreTask;
|
||||
try {
|
||||
task = await adapter.runTemplate(input.projectId, input.templateId, { limit: input.limit, message: `Update run for ${server.name}, started in Homelab Manager by ${input.actor} (limited to ${input.limit})` });
|
||||
task = await adapter.runTemplate(input.projectId, input.templateId, { limit: input.limit, message: `${kind === "restart" ? "Restart" : "Update run"} for ${server.name}, started in Homelab Manager by ${input.actor} (limited to ${input.limit})` });
|
||||
} catch (err) {
|
||||
if (started && !started.extended) await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(eq(maintenanceWindows.id, started.window.id));
|
||||
invalidateAlertsCache();
|
||||
@@ -136,9 +157,13 @@ export async function startUpdateRun(input: StartInput, now: Date = new Date()):
|
||||
startedBy: input.actor,
|
||||
startedAt: now.toISOString(),
|
||||
updatesBefore: updates?.count ?? null,
|
||||
kind,
|
||||
restartProjectId: restart?.projectId ?? null,
|
||||
restartTemplateId: restart?.templateId ?? null,
|
||||
restartTemplateName,
|
||||
})
|
||||
.returning();
|
||||
await recordAudit({ category: "update_run", action: "start", targetType: "server", targetId: server.id, detail: { run: run.id, template: templateName, limit: input.limit, task: task.id, by: input.actor } });
|
||||
await recordAudit({ category: "update_run", action: "start", targetType: "server", targetId: server.id, detail: { run: run.id, kind, template: templateName, limit: input.limit, task: task.id, by: input.actor, ...(restartTemplateName ? { restartTemplate: restartTemplateName } : {}) } });
|
||||
return run;
|
||||
}
|
||||
|
||||
@@ -160,23 +185,114 @@ async function finish(run: UpdateRunRow, status: "done" | "failed" | "cancelled"
|
||||
const [server] = await db.select({ name: servers.name }).from(servers).where(eq(servers.id, run.serverId)).limit(1);
|
||||
await recordAudit({ category: "update_run", action: status === "done" ? "finished" : status, targetType: "server", targetId: run.serverId, detail: { run: run.id, template: run.templateName, message } });
|
||||
if (status !== "cancelled") {
|
||||
await notifyUpdateRunFinished({ serverName: server?.name ?? `#${run.serverId}`, templateName: run.templateName, ok: status === "done", message }).catch((err) => console.error("[updateRuns] couldn't send the result:", err));
|
||||
await notifyUpdateRunFinished({ serverName: server?.name ?? `#${run.serverId}`, templateName: run.templateName, ok: status === "done", message, kind: run.kind === "restart" ? "restart" : "update" }).catch((err) => console.error("[updateRuns] couldn't send the result:", err));
|
||||
}
|
||||
}
|
||||
|
||||
const updatesWords = (n: number) => (n === 1 ? "1 update" : `${n} updates`);
|
||||
|
||||
/** What is left of the updates, in a clause ("2 updates still waiting (10 updates installed)"). */
|
||||
function updatesClause(before: number | null, after: number | null): string {
|
||||
if (after === null) return "The server reported again, but doesn't say what is waiting";
|
||||
let words = after === 0 ? "No updates are waiting now" : `${updatesWords(after)} still waiting`;
|
||||
if (before !== null && before > after) words += ` (${updatesWords(before - after)} installed)`;
|
||||
return words;
|
||||
}
|
||||
|
||||
/** What a finished run says about the server, from the report that came after it. */
|
||||
export function summarizeOutcome(before: number | null, after: number | null, reboot: boolean): string {
|
||||
const parts: string[] = [];
|
||||
if (after !== null) {
|
||||
parts.push(after === 0 ? "No updates are waiting now" : `${updatesWords(after)} still waiting`);
|
||||
if (before !== null && before > after) parts[0] += ` (${updatesWords(before - after)} installed)`;
|
||||
} else {
|
||||
parts.push("The server reported again, but doesn't say what is waiting");
|
||||
return `${[updatesClause(before, after), reboot ? "a restart is still needed" : "no restart is needed"].join("; ")}.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the server has started since `sinceMs`, from the uptime in its last report (its start is the report's time minus its uptime): true or false, or null when it doesn't say.
|
||||
* A start a little before `sinceMs` still counts (see BOOT_SLACK_MS).
|
||||
*/
|
||||
export function startedSince(server: { lastSeenAt: string | null; uptimeSeconds: number | null }, sinceMs: number): boolean | null {
|
||||
if (!server.lastSeenAt || server.uptimeSeconds === null || server.uptimeSeconds === undefined) return null;
|
||||
const seen = Date.parse(server.lastSeenAt);
|
||||
if (Number.isNaN(seen)) return null;
|
||||
return seen - server.uptimeSeconds * 1000 >= sinceMs - BOOT_SLACK_MS;
|
||||
}
|
||||
|
||||
export interface RestartOutcome {
|
||||
/** The server is taken to have started again. */
|
||||
restarted: boolean;
|
||||
/** Something said so (its uptime, or its "restart needed" flag clearing); false means it only reported again and nothing says whether it restarted. */
|
||||
confirmed: boolean;
|
||||
}
|
||||
|
||||
/** Did the restart happen? Its uptime says so where it is reported; failing that, the "restart needed" flag clearing; failing that, nothing can tell. */
|
||||
export function restartOutcome(server: { lastSeenAt: string | null; uptimeSeconds: number | null }, updates: { rebootRequired: boolean } | null, sinceMs: number): RestartOutcome {
|
||||
const booted = startedSince(server, sinceMs);
|
||||
if (booted !== null) return { restarted: booted, confirmed: true };
|
||||
if (updates) return { restarted: !updates.rebootRequired, confirmed: true };
|
||||
return { restarted: true, confirmed: false };
|
||||
}
|
||||
|
||||
/** What a finished restart says: for an update run, what is left of the updates and then the restart; for a restart on its own, only the restart. */
|
||||
export function summarizeRestart(kind: UpdateRunKind, before: number | null, after: number | null, outcome: RestartOutcome, stillDue: boolean): string {
|
||||
const lead = kind === "update" ? `${updatesClause(before, after)}; ` : "";
|
||||
const subject = kind === "update" ? "the server" : "The server";
|
||||
if (!outcome.restarted) {
|
||||
return `${lead}${kind === "update" ? "the restart playbook finished" : "The playbook finished"}, but the server has not started again — check that the playbook really restarts it.`;
|
||||
}
|
||||
parts.push(reboot ? "a restart is still needed" : "no restart is needed");
|
||||
return `${parts.join("; ")}.`;
|
||||
const how = outcome.confirmed ? "" : " (its uptime isn't reported, so that couldn't be confirmed)";
|
||||
return `${lead}${subject} was restarted and is back${stillDue ? ", but says another restart is due" : ""}${how}.`;
|
||||
}
|
||||
|
||||
/** Keeps the quiet up for at least `minutes` more; a window that has ended is made again, since the server is about to restart. Returns what changed on the run, if anything. */
|
||||
async function keepQuiet(run: UpdateRunRow, minutes: number, now: Date): Promise<Partial<UpdateRunRow>> {
|
||||
const until = new Date(now.getTime() + minutes * 60_000).toISOString();
|
||||
if (run.windowId !== null) {
|
||||
const [w] = await db.select().from(maintenanceWindows).where(eq(maintenanceWindows.id, run.windowId)).limit(1);
|
||||
if (w && w.endsAt > now.toISOString()) {
|
||||
if (w.endsAt < until) {
|
||||
await db.update(maintenanceWindows).set({ endsAt: until }).where(eq(maintenanceWindows.id, w.id));
|
||||
invalidateAlertsCache();
|
||||
}
|
||||
return {};
|
||||
}
|
||||
}
|
||||
const started = await startOrExtendWindow("server", run.serverId, minutes, `Update run: restart`, run.startedBy, now);
|
||||
invalidateAlertsCache();
|
||||
return started ? { windowId: started.window.id, windowCreated: !started.extended } : {};
|
||||
}
|
||||
|
||||
/** The updates are in and the server says a restart is due: run the restart playbook. If it can't be started, the run ends and says the updates are in. */
|
||||
async function beginRestart(run: UpdateRunRow, server: typeof servers.$inferSelect, updates: { count: number; rebootRequired: boolean } | null, now: Date): Promise<void> {
|
||||
const seen = { updatesAfter: updates?.count ?? null, rebootRequired: true, hostReportedAt: server.lastSeenAt };
|
||||
const adapter = await semaphoreFor(run.integrationId);
|
||||
const clause = updatesClause(run.updatesBefore, updates?.count ?? null);
|
||||
if (!adapter || run.restartProjectId === null || run.restartTemplateId === null) {
|
||||
await finish(run, "failed", `${clause}, but the restart could not be started: Semaphore isn't available. The server still needs a restart.`, now, seen);
|
||||
return;
|
||||
}
|
||||
const quiet = await keepQuiet(run, RESTART_WINDOW_MINUTES, now);
|
||||
let task: SemaphoreTask;
|
||||
try {
|
||||
task = await adapter.runTemplate(run.restartProjectId, run.restartTemplateId, { limit: run.hostLimit, message: `Restart of ${server.name} after update run #${run.id}, started in Homelab Manager (limited to ${run.hostLimit})` });
|
||||
} catch (err) {
|
||||
await finish(run, "failed", `${clause}, but the restart could not be started: ${err instanceof Error ? err.message : String(err)}. The server still needs a restart.`, now, seen);
|
||||
return;
|
||||
}
|
||||
await db
|
||||
.update(updateRuns)
|
||||
.set({ ...seen, ...quiet, status: "restarting", restartTaskId: task.id, restartTaskStatus: task.status, restartStartedAt: now.toISOString(), pollErrors: 0, message: `${clause}. The server says a restart is due — restarting it with "${run.restartTemplateName ?? "the restart playbook"}".` })
|
||||
.where(eq(updateRuns.id, run.id));
|
||||
await recordAudit({ category: "update_run", action: "restart", targetType: "server", targetId: run.serverId, detail: { run: run.id, template: run.restartTemplateName, task: task.id } });
|
||||
}
|
||||
|
||||
/** A restart is over and the server has reported again: did it start again? */
|
||||
async function finishRestart(run: UpdateRunRow, server: typeof servers.$inferSelect, updates: ReturnType<typeof parsePendingUpdates>, sinceMs: number, now: Date): Promise<void> {
|
||||
const outcome = restartOutcome(server, updates, sinceMs);
|
||||
const stillDue = updates?.rebootRequired ?? false;
|
||||
await finish(run, outcome.restarted ? "done" : "failed", summarizeRestart(run.kind === "restart" ? "restart" : "update", run.updatesBefore, updates?.count ?? null, outcome, outcome.restarted && stillDue), now, {
|
||||
hostReportedAt: server.lastSeenAt,
|
||||
updatesAfter: updates?.count ?? null,
|
||||
rebootRequired: stillDue,
|
||||
restarted: outcome.restarted,
|
||||
});
|
||||
}
|
||||
|
||||
async function advance(run: UpdateRunRow, now: Date): Promise<void> {
|
||||
@@ -185,11 +301,15 @@ async function advance(run: UpdateRunRow, now: Date): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
if (run.status === "running") {
|
||||
// The playbook is being followed: the update's, or (the second phase) the one that restarts the server.
|
||||
if (run.status === "running" || run.status === "restarting") {
|
||||
const restarting = run.status === "restarting";
|
||||
const projectId = restarting ? run.restartProjectId : run.projectId;
|
||||
const taskId = restarting ? run.restartTaskId : run.taskId;
|
||||
const adapter = await semaphoreFor(run.integrationId);
|
||||
let task: SemaphoreTask | null = null;
|
||||
try {
|
||||
task = adapter && run.taskId !== null ? await adapter.getTask(run.projectId, run.taskId) : null;
|
||||
task = adapter && projectId !== null && taskId !== null ? await adapter.getTask(projectId, taskId) : null;
|
||||
} catch {
|
||||
task = null;
|
||||
}
|
||||
@@ -199,22 +319,27 @@ async function advance(run: UpdateRunRow, now: Date): Promise<void> {
|
||||
else await db.update(updateRuns).set({ pollErrors: errors }).where(eq(updateRuns.id, run.id));
|
||||
return;
|
||||
}
|
||||
const lastStatus = restarting ? run.restartTaskStatus : run.taskStatus;
|
||||
const statusField = restarting ? { restartTaskStatus: task.status } : { taskStatus: task.status };
|
||||
if (!FINISHED_TASK.has(task.status)) {
|
||||
if (task.status !== run.taskStatus || run.pollErrors > 0) await db.update(updateRuns).set({ taskStatus: task.status, pollErrors: 0 }).where(eq(updateRuns.id, run.id));
|
||||
if (task.status !== lastStatus || run.pollErrors > 0) await db.update(updateRuns).set({ ...statusField, pollErrors: 0 }).where(eq(updateRuns.id, run.id));
|
||||
return;
|
||||
}
|
||||
if (task.status === "success") {
|
||||
const ended = task.end ? new Date(task.end).toISOString() : now.toISOString();
|
||||
await db.update(updateRuns).set({ status: "waiting_for_host", taskStatus: task.status, taskEndedAt: ended, pollErrors: 0, message: "The playbook finished. Waiting for the server to report again." }).where(eq(updateRuns.id, run.id));
|
||||
if (restarting) await db.update(updateRuns).set({ status: "waiting_for_restart", ...statusField, restartEndedAt: ended, pollErrors: 0, message: "The restart playbook finished. Waiting for the server to come back and report." }).where(eq(updateRuns.id, run.id));
|
||||
else await db.update(updateRuns).set({ status: "waiting_for_host", ...statusField, taskEndedAt: ended, pollErrors: 0, message: run.kind === "restart" ? "The restart playbook finished. Waiting for the server to come back and report." : "The playbook finished. Waiting for the server to report again." }).where(eq(updateRuns.id, run.id));
|
||||
return;
|
||||
}
|
||||
const why = task.status === "error" ? "The playbook failed in Semaphore" : task.status === "rejected" ? "The run was rejected in Semaphore" : "The run was stopped in Semaphore";
|
||||
await finish(run, "failed", `${why} (run #${run.taskId}). The server is no longer under maintenance.`, now, { taskStatus: task.status });
|
||||
if (restarting) await finish(run, "failed", `The updates are in, but the restart didn't happen. ${why} (run #${taskId}). The server still needs a restart, and is no longer under maintenance.`, now, statusField);
|
||||
else await finish(run, "failed", `${why} (run #${taskId}). The server is no longer under maintenance.`, now, statusField);
|
||||
return;
|
||||
}
|
||||
|
||||
// waiting_for_host: the agent has reported since the playbook ended (and, where it reports updates, those are newer than the run too).
|
||||
const ended = Date.parse(run.taskEndedAt ?? run.startedAt);
|
||||
// Waiting for the server: the agent has reported since the playbook ended (and, where it reports updates, those are newer than the run too).
|
||||
const afterRestart = run.status === "waiting_for_restart";
|
||||
const ended = Date.parse((afterRestart ? run.restartEndedAt : run.taskEndedAt) ?? (afterRestart ? run.restartStartedAt : null) ?? run.startedAt);
|
||||
const [server] = await db.select().from(servers).where(eq(servers.id, run.serverId)).limit(1);
|
||||
if (!server) {
|
||||
await finish(run, "failed", "The server was removed.", now);
|
||||
@@ -224,15 +349,19 @@ async function advance(run: UpdateRunRow, now: Date): Promise<void> {
|
||||
const updates = parsePendingUpdates(server.pendingUpdates);
|
||||
const updatesFresh = updates ? Date.parse(updates.reportedAt) > ended : true;
|
||||
if (reported && updatesFresh) {
|
||||
await finish(run, "done", summarizeOutcome(run.updatesBefore, updates?.count ?? null, updates?.rebootRequired ?? false), now, {
|
||||
hostReportedAt: server.lastSeenAt,
|
||||
updatesAfter: updates?.count ?? null,
|
||||
rebootRequired: updates?.rebootRequired ?? false,
|
||||
});
|
||||
if (afterRestart) await finishRestart(run, server, updates, Date.parse(run.restartStartedAt ?? run.startedAt), now);
|
||||
else if (run.kind === "restart") await finishRestart(run, server, updates, Date.parse(run.startedAt), now);
|
||||
else if (run.restartTemplateId !== null && updates?.rebootRequired) await beginRestart(run, server, updates, now);
|
||||
else
|
||||
await finish(run, "done", summarizeOutcome(run.updatesBefore, updates?.count ?? null, updates?.rebootRequired ?? false), now, {
|
||||
hostReportedAt: server.lastSeenAt,
|
||||
updatesAfter: updates?.count ?? null,
|
||||
rebootRequired: updates?.rebootRequired ?? false,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (now.getTime() - ended > HOST_WAIT_MS) {
|
||||
await finish(run, "failed", `The playbook finished, but the server hasn't reported again in ${Math.round(HOST_WAIT_MS / 60_000)} minutes. Check that it came back.`, now);
|
||||
await finish(run, "failed", `The ${afterRestart || run.kind === "restart" ? "restart playbook" : "playbook"} finished, but the server hasn't reported again in ${Math.round(HOST_WAIT_MS / 60_000)} minutes. Check that it came back.`, now);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,569 @@
|
||||
/**
|
||||
* Restarting a server after its updates: a second playbook, run when the server says a restart is due; the quiet kept up for it; the server waited for and checked
|
||||
* to have started again (from its uptime, or its "restart needed" flag clearing); a restart on its own; and a patch plan that restarts what needs it.
|
||||
* Semaphore is a stand-in that answers as told.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-restart-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, updateRuns, auditLog, maintenanceWindows, patchPlans, patchPlanRuns } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { startUpdateRun, advanceUpdateRuns, setSemaphoreForTests, startedSince, restartOutcome, summarizeRestart, UpdateRunError, HOST_WAIT_MS, RESTART_WINDOW_MINUTES, ACTIVE_STATUSES } = await import("../src/services/updateRuns.js");
|
||||
const { startPlanRun, advancePlanRuns, parseItems } = await import("../src/services/patchPlans.js");
|
||||
const { listActiveWindows } = await import("../src/services/maintenance.js");
|
||||
const { getServerActivity } = await import("../src/services/serverActivity.js");
|
||||
const { updateRunsRouter } = await import("../src/routes/updateRuns.js");
|
||||
const { patchPlansRouter } = await import("../src/routes/patchPlans.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
// ── a Semaphore that answers as told: two templates (updates, restart), and a task for every run ──
|
||||
interface Task {
|
||||
id: number;
|
||||
status: string;
|
||||
created: string | null;
|
||||
start: string | null;
|
||||
end: string | null;
|
||||
}
|
||||
const fake = { runs: [] as { projectId: number; templateId: number; limit?: string; message?: string }[], tasks: new Map<number, Task>(), next: 100, refuseTemplate: null as number | null };
|
||||
const resetFake = () => {
|
||||
fake.runs = [];
|
||||
fake.tasks = new Map();
|
||||
fake.next = 100;
|
||||
fake.refuseTemplate = null;
|
||||
};
|
||||
const setTask = (id: number, patch: Partial<Task>) => fake.tasks.set(id, { ...fake.tasks.get(id)!, ...patch });
|
||||
setSemaphoreForTests(async (integrationId) =>
|
||||
integrationId === 9
|
||||
? {
|
||||
ping: async () => ({ ok: true }),
|
||||
listTemplatesWithStatus: async () => [],
|
||||
checkTemplates: async () => ({
|
||||
templates: [
|
||||
{ id: 7, projectId: 2, projectName: "Homelab", name: "Apply updates", playbook: "updates.yml", lastTask: null },
|
||||
{ id: 8, projectId: 2, projectName: "Homelab", name: "Restart server", playbook: "restart.yml", lastTask: null },
|
||||
],
|
||||
failedProjectIds: [],
|
||||
}),
|
||||
runTemplate: async (projectId, templateId, options) => {
|
||||
if (fake.refuseTemplate === templateId) throw new Error("Semaphore is having a moment");
|
||||
fake.runs.push({ projectId, templateId, limit: options?.limit, message: options?.message });
|
||||
const task = { id: fake.next++, status: "waiting", created: null, start: null, end: null };
|
||||
fake.tasks.set(task.id, task);
|
||||
return { ...task } as never;
|
||||
},
|
||||
getTask: async (_p, id) => ({ ...fake.tasks.get(id)! }) as never,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
|
||||
const T0 = new Date("2026-10-17T12:00:00.000Z");
|
||||
const at = (minutes: number) => new Date(T0.getTime() + minutes * 60_000);
|
||||
const iso = (minutes: number) => at(minutes).toISOString();
|
||||
const RESTART = { projectId: 2, templateId: 8 };
|
||||
const input = (serverId: number, extra: object = {}) => ({ serverId, integrationId: 9, projectId: 2, templateId: 7, limit: "elsa", windowMinutes: 90, actor: "Olle", ...extra });
|
||||
const runRow = async (id: number) => (await db.select().from(updateRuns).where(eq(updateRuns.id, id)))[0];
|
||||
const updatesJson = (count: number, reboot: boolean, minutes: number) => JSON.stringify({ manager: "apt", count, security: 0, rebootRequired: reboot, reportedAt: iso(minutes) });
|
||||
|
||||
describe("restarting after the updates", () => {
|
||||
let serverId = 0;
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "admin", name: "Ada", role: "admin" }]);
|
||||
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: iso(0), uptimeSeconds: 86_400 * 30, pendingUpdates: updatesJson(12, false, 0) }).returning();
|
||||
serverId = s.id;
|
||||
});
|
||||
beforeEach(async () => {
|
||||
resetFake();
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
await db.update(servers).set({ lastSeenAt: iso(0), uptimeSeconds: 86_400 * 30, pendingUpdates: updatesJson(12, false, 0) }).where(eq(servers.id, serverId));
|
||||
});
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
/** Reports as the agent would: when, what is waiting, whether a restart is due, and how long it has been up. */
|
||||
const report = (minutes: number, count: number, reboot: boolean, uptimeSeconds: number | null) =>
|
||||
db.update(servers).set({ lastSeenAt: iso(minutes), uptimeSeconds, pendingUpdates: updatesJson(count, reboot, minutes) }).where(eq(servers.id, serverId));
|
||||
|
||||
/** Runs an update with a restart playbook to the point where the server says a restart is due. */
|
||||
async function toRestart(start = 0) {
|
||||
const run = await startUpdateRun(input(serverId, { restart: RESTART }), at(start));
|
||||
setTask(run.taskId!, { status: "success", end: iso(start + 8) });
|
||||
await advanceUpdateRuns(at(start + 9));
|
||||
await report(start + 14, 2, true, 86_400 * 30 + 840);
|
||||
await advanceUpdateRuns(at(start + 15));
|
||||
return run;
|
||||
}
|
||||
|
||||
test("an update run remembers the restart playbook, says so in its history, and refuses one that isn't in Semaphore", async () => {
|
||||
const run = await startUpdateRun(input(serverId, { restart: RESTART }), T0);
|
||||
assert.equal(run.kind, "update");
|
||||
assert.deepEqual([run.restartProjectId, run.restartTemplateId, run.restartTemplateName], [2, 8, "Restart server"]);
|
||||
assert.equal(fake.runs.length, 1, "only the update playbook is started now");
|
||||
const events = await getServerActivity(serverId, at(1));
|
||||
assert.ok(events.some((e) => e.title === "Update run started" && /then a restart with Restart server if one is due/.test(e.detail ?? "")));
|
||||
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
fake.runs = [];
|
||||
await assert.rejects(startUpdateRun(input(serverId, { restart: { projectId: 2, templateId: 999 } }), T0), (e: unknown) => e instanceof UpdateRunError && e.status === 404 && e.code === "restart_template_not_found");
|
||||
assert.equal(fake.runs.length, 0, "nothing was started");
|
||||
assert.equal((await listActiveWindows(T0)).length, 0, "and no quiet was left behind");
|
||||
});
|
||||
|
||||
test("when the server says a restart is due, the restart playbook runs, is followed, and the server is checked to have started again", async () => {
|
||||
const run = await toRestart();
|
||||
let r = await runRow(run.id);
|
||||
assert.equal(r.status, "restarting");
|
||||
assert.equal(r.updatesAfter, 2);
|
||||
assert.equal(r.restartStartedAt, iso(15));
|
||||
assert.deepEqual(fake.runs.map((x) => [x.templateId, x.limit]), [[7, "elsa"], [8, "elsa"]], "the same server, and only that one");
|
||||
assert.match(fake.runs[1].message!, /Restart of elsa after update run #\d+/);
|
||||
assert.match(r.message!, /2 updates still waiting \(10 updates installed\)\. The server says a restart is due — restarting it with "Restart server"/);
|
||||
assert.ok(ACTIVE_STATUSES.includes(r.status as never), "it counts as under way");
|
||||
assert.ok((await db.select().from(auditLog)).some((a) => a.category === "update_run" && a.action === "restart"));
|
||||
|
||||
// the quiet is kept up for the restart (it was the update's 90 minutes; the restart's own is shorter, so it stays at 90)
|
||||
const windows = await listActiveWindows(at(16));
|
||||
assert.equal(windows.length, 1);
|
||||
assert.equal(windows[0].id, r.windowId);
|
||||
|
||||
setTask(r.restartTaskId!, { status: "running" });
|
||||
await advanceUpdateRuns(at(17));
|
||||
r = await runRow(run.id);
|
||||
assert.equal(r.status, "restarting");
|
||||
assert.equal(r.restartTaskStatus, "running");
|
||||
assert.equal(r.taskStatus, "success", "the update's own status is left as it was");
|
||||
|
||||
setTask(r.restartTaskId!, { status: "success", end: iso(19) });
|
||||
await advanceUpdateRuns(at(20));
|
||||
r = await runRow(run.id);
|
||||
assert.equal(r.status, "waiting_for_restart");
|
||||
assert.equal(r.restartEndedAt, iso(19));
|
||||
assert.match(r.message!, /Waiting for the server to come back/);
|
||||
|
||||
await advanceUpdateRuns(at(22));
|
||||
assert.equal((await runRow(run.id)).status, "waiting_for_restart", "the report from before the restart ended doesn't count");
|
||||
|
||||
await report(23, 2, false, 150); // up for two and a half minutes: it started at 20:30, after the restart began
|
||||
await advanceUpdateRuns(at(24));
|
||||
r = await runRow(run.id);
|
||||
assert.equal(r.status, "done");
|
||||
assert.equal(r.restarted, true);
|
||||
assert.equal(r.rebootRequired, false);
|
||||
assert.equal(r.message, "2 updates still waiting (10 updates installed); the server was restarted and is back.");
|
||||
assert.equal((await listActiveWindows(at(25))).length, 0, "the window it made is over");
|
||||
});
|
||||
|
||||
test("a server that says no restart is due is not restarted", async () => {
|
||||
const run = await startUpdateRun(input(serverId, { restart: RESTART }), T0);
|
||||
setTask(run.taskId!, { status: "success", end: iso(8) });
|
||||
await advanceUpdateRuns(at(9));
|
||||
await report(14, 0, false, 600);
|
||||
await advanceUpdateRuns(at(15));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "done");
|
||||
assert.equal(r.restarted, false);
|
||||
assert.equal(r.message, "No updates are waiting now (12 updates installed); no restart is needed.");
|
||||
assert.equal(fake.runs.length, 1, "the restart playbook was never run");
|
||||
});
|
||||
|
||||
test("without a restart playbook a restart that is due is only said to be, as before", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
setTask(run.taskId!, { status: "success", end: iso(8) });
|
||||
await advanceUpdateRuns(at(9));
|
||||
await report(14, 2, true, 840);
|
||||
await advanceUpdateRuns(at(15));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "done");
|
||||
assert.equal(r.message, "2 updates still waiting (10 updates installed); a restart is still needed.");
|
||||
assert.equal(fake.runs.length, 1);
|
||||
});
|
||||
|
||||
test("if the restart playbook fails, the run says the updates are in and the server still needs a restart; the quiet ends", async () => {
|
||||
const run = await toRestart();
|
||||
const { restartTaskId } = await runRow(run.id);
|
||||
setTask(restartTaskId!, { status: "error" });
|
||||
await advanceUpdateRuns(at(18));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "failed");
|
||||
assert.match(r.message!, /The updates are in, but the restart didn't happen\. The playbook failed in Semaphore \(run #\d+\)\. The server still needs a restart/);
|
||||
assert.equal((await listActiveWindows(at(19))).length, 0);
|
||||
});
|
||||
|
||||
test("if the restart playbook can't be started, the run ends saying so, with the updates counted", async () => {
|
||||
fake.refuseTemplate = 8;
|
||||
const run = await startUpdateRun(input(serverId, { restart: RESTART }), T0);
|
||||
setTask(run.taskId!, { status: "success", end: iso(8) });
|
||||
await advanceUpdateRuns(at(9));
|
||||
await report(14, 2, true, 840);
|
||||
await advanceUpdateRuns(at(15));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "failed");
|
||||
assert.equal(r.message, "2 updates still waiting (10 updates installed), but the restart could not be started: Semaphore is having a moment. The server still needs a restart.");
|
||||
assert.equal(r.rebootRequired, true);
|
||||
assert.equal((await listActiveWindows(at(16))).length, 0);
|
||||
});
|
||||
|
||||
test("a playbook that finished without the server restarting is a failure, from its uptime — and a restart flag that never clears is too", async () => {
|
||||
// reported again, but it has been up since long before the restart began
|
||||
let run = await toRestart();
|
||||
setTask((await runRow(run.id)).restartTaskId!, { status: "success", end: iso(19) });
|
||||
await advanceUpdateRuns(at(20));
|
||||
await report(23, 2, false, 86_400 * 30 + 1380);
|
||||
await advanceUpdateRuns(at(24));
|
||||
let r = await runRow(run.id);
|
||||
assert.equal(r.status, "failed");
|
||||
assert.equal(r.restarted, false);
|
||||
assert.equal(r.message, "2 updates still waiting (10 updates installed); the restart playbook finished, but the server has not started again — check that the playbook really restarts it.");
|
||||
|
||||
// no uptime in its reports: the restart flag is what there is
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
run = await toRestart(100);
|
||||
setTask((await runRow(run.id)).restartTaskId!, { status: "success", end: iso(119) });
|
||||
await advanceUpdateRuns(at(120));
|
||||
await report(123, 2, true, null);
|
||||
await advanceUpdateRuns(at(124));
|
||||
r = await runRow(run.id);
|
||||
assert.equal(r.status, "failed", "still says a restart is due");
|
||||
assert.match(r.message!, /has not started again/);
|
||||
});
|
||||
|
||||
test("with no uptime to go by, the flag clearing is enough; with neither, it is taken to have restarted and says it couldn't be confirmed", async () => {
|
||||
let run = await toRestart();
|
||||
setTask((await runRow(run.id)).restartTaskId!, { status: "success", end: iso(19) });
|
||||
await advanceUpdateRuns(at(20));
|
||||
await report(23, 2, false, null);
|
||||
await advanceUpdateRuns(at(24));
|
||||
let r = await runRow(run.id);
|
||||
assert.equal(r.status, "done");
|
||||
assert.equal(r.message, "2 updates still waiting (10 updates installed); the server was restarted and is back.");
|
||||
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
run = await toRestart(100);
|
||||
setTask((await runRow(run.id)).restartTaskId!, { status: "success", end: iso(119) });
|
||||
await advanceUpdateRuns(at(120));
|
||||
await db.update(servers).set({ lastSeenAt: iso(123), uptimeSeconds: null, pendingUpdates: null }).where(eq(servers.id, serverId));
|
||||
await advanceUpdateRuns(at(124));
|
||||
r = await runRow(run.id);
|
||||
assert.equal(r.status, "done");
|
||||
assert.match(r.message!, /was restarted and is back \(its uptime isn't reported, so that couldn't be confirmed\)\.$/);
|
||||
});
|
||||
|
||||
test("a restarted server that says another restart is due is done, with that said", async () => {
|
||||
const run = await toRestart();
|
||||
setTask((await runRow(run.id)).restartTaskId!, { status: "success", end: iso(19) });
|
||||
await advanceUpdateRuns(at(20));
|
||||
await report(23, 0, true, 150);
|
||||
await advanceUpdateRuns(at(24));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "done");
|
||||
assert.equal(r.rebootRequired, true);
|
||||
assert.match(r.message!, /the server was restarted and is back, but says another restart is due\.$/);
|
||||
});
|
||||
|
||||
test("a server that doesn't come back from its restart is given up on, and the quiet ends", async () => {
|
||||
const run = await toRestart();
|
||||
setTask((await runRow(run.id)).restartTaskId!, { status: "success", end: iso(19) });
|
||||
await advanceUpdateRuns(at(20));
|
||||
await advanceUpdateRuns(at(19 + HOST_WAIT_MS / 60_000 + 1));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "failed");
|
||||
assert.match(r.message!, /The restart playbook finished, but the server hasn't reported again in 30 minutes/);
|
||||
assert.equal((await listActiveWindows(at(60))).length, 0);
|
||||
});
|
||||
|
||||
test("the quiet is made again if it ended before the restart began, and is not shortened if it is longer than the restart needs", async () => {
|
||||
// a window longer than the restart needs (somebody made it longer while the update ran) is not cut short
|
||||
const run = await startUpdateRun(input(serverId, { restart: RESTART }), T0);
|
||||
await db.update(maintenanceWindows).set({ endsAt: iso(600) }).where(eq(maintenanceWindows.id, run.windowId!));
|
||||
setTask(run.taskId!, { status: "success", end: iso(8) });
|
||||
await advanceUpdateRuns(at(9));
|
||||
await report(14, 2, true, 840);
|
||||
await advanceUpdateRuns(at(15));
|
||||
assert.equal((await runRow(run.id)).status, "restarting");
|
||||
assert.equal((await listActiveWindows(at(16)))[0].endsAt, iso(600), "left as long as it was");
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
|
||||
// one that has already run out is made again, for the restart
|
||||
const second = await startUpdateRun(input(serverId, { restart: RESTART, windowMinutes: 15 }), at(200));
|
||||
setTask(second.taskId!, { status: "success", end: iso(208) });
|
||||
await advanceUpdateRuns(at(209));
|
||||
await report(214, 2, true, 600);
|
||||
await advanceUpdateRuns(at(216)); // the 15 minutes ran out at 215
|
||||
const r = await runRow(second.id);
|
||||
assert.equal(r.status, "restarting");
|
||||
const windows = await listActiveWindows(at(217));
|
||||
assert.equal(windows.length, 1, "quiet again for the restart");
|
||||
assert.equal(Date.parse(windows[0].endsAt) - at(216).getTime(), RESTART_WINDOW_MINUTES * 60_000);
|
||||
assert.equal(r.windowId, windows[0].id);
|
||||
});
|
||||
});
|
||||
|
||||
describe("restarting a server on its own", () => {
|
||||
let serverId = 0;
|
||||
before(async () => {
|
||||
const [s] = await db.insert(servers).values({ name: "anna", apiTokenHash: hashToken("tok2"), apiTokenPrefix: "t2", lastSeenAt: iso(0), uptimeSeconds: 86_400 * 10, pendingUpdates: updatesJson(0, true, 0) }).returning();
|
||||
serverId = s.id;
|
||||
});
|
||||
beforeEach(async () => {
|
||||
resetFake();
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
await db.update(servers).set({ lastSeenAt: iso(0), uptimeSeconds: 86_400 * 10, pendingUpdates: updatesJson(0, true, 0) }).where(eq(servers.id, serverId));
|
||||
});
|
||||
|
||||
test("a restart run runs the playbook it was given, follows it, and checks the server started again", async () => {
|
||||
const run = await startUpdateRun(input(serverId, { kind: "restart", templateId: 8, limit: "anna", restart: RESTART }), T0);
|
||||
assert.equal(run.kind, "restart");
|
||||
assert.equal(run.templateName, "Restart server");
|
||||
assert.equal(run.restartTemplateId, null, "a restart doesn't restart again");
|
||||
assert.deepEqual(fake.runs.map((x) => [x.templateId, x.limit]), [[8, "anna"]]);
|
||||
assert.match(fake.runs[0].message!, /^Restart for anna, started in Homelab Manager by Olle/);
|
||||
const windows = await listActiveWindows(T0);
|
||||
assert.equal(windows[0].reason, "Restart: Restart server");
|
||||
|
||||
setTask(run.taskId!, { status: "success", end: iso(4) });
|
||||
await advanceUpdateRuns(at(5));
|
||||
let r = await runRow(run.id);
|
||||
assert.equal(r.status, "waiting_for_host");
|
||||
assert.match(r.message!, /The restart playbook finished\. Waiting for the server to come back/);
|
||||
|
||||
await db.update(servers).set({ lastSeenAt: iso(9), uptimeSeconds: 200, pendingUpdates: updatesJson(0, false, 9) }).where(eq(servers.id, serverId));
|
||||
await advanceUpdateRuns(at(10));
|
||||
r = await runRow(run.id);
|
||||
assert.equal(r.status, "done");
|
||||
assert.equal(r.restarted, true);
|
||||
assert.equal(r.message, "The server was restarted and is back.");
|
||||
assert.equal((await listActiveWindows(at(11))).length, 0);
|
||||
});
|
||||
|
||||
test("a restart that did not happen fails, in its own words", async () => {
|
||||
const run = await startUpdateRun(input(serverId, { kind: "restart", templateId: 8, limit: "anna" }), T0);
|
||||
setTask(run.taskId!, { status: "success", end: iso(4) });
|
||||
await advanceUpdateRuns(at(5));
|
||||
await db.update(servers).set({ lastSeenAt: iso(9), uptimeSeconds: 86_400 * 10 + 540, pendingUpdates: updatesJson(0, true, 9) }).where(eq(servers.id, serverId));
|
||||
await advanceUpdateRuns(at(10));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "failed");
|
||||
assert.equal(r.message, "The playbook finished, but the server has not started again — check that the playbook really restarts it.");
|
||||
});
|
||||
|
||||
test("it is in the server's history as a restart", async () => {
|
||||
await startUpdateRun(input(serverId, { kind: "restart", templateId: 8, limit: "anna" }), T0);
|
||||
const events = await getServerActivity(serverId, at(1));
|
||||
assert.ok(events.some((e) => e.title === "Restart started"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("working out whether a server has started again", () => {
|
||||
const since = T0.getTime();
|
||||
test("from its uptime: the report's time minus the uptime is when it started", () => {
|
||||
assert.equal(startedSince({ lastSeenAt: iso(10), uptimeSeconds: 300 }, since), true, "started at 12:05");
|
||||
assert.equal(startedSince({ lastSeenAt: iso(10), uptimeSeconds: 86_400 }, since), false);
|
||||
assert.equal(startedSince({ lastSeenAt: iso(10), uptimeSeconds: 10 * 60 + 60 }, since), true, "a minute before the restart began still counts");
|
||||
assert.equal(startedSince({ lastSeenAt: iso(10), uptimeSeconds: 10 * 60 + 600 }, since), false, "ten minutes before does not");
|
||||
assert.equal(startedSince({ lastSeenAt: iso(10), uptimeSeconds: null }, since), null);
|
||||
assert.equal(startedSince({ lastSeenAt: null, uptimeSeconds: 100 }, since), null);
|
||||
});
|
||||
|
||||
test("the uptime is believed over the flag; the flag over nothing", () => {
|
||||
assert.deepEqual(restartOutcome({ lastSeenAt: iso(10), uptimeSeconds: 300 }, { rebootRequired: true }, since), { restarted: true, confirmed: true });
|
||||
assert.deepEqual(restartOutcome({ lastSeenAt: iso(10), uptimeSeconds: 86_400 }, { rebootRequired: false }, since), { restarted: false, confirmed: true });
|
||||
assert.deepEqual(restartOutcome({ lastSeenAt: iso(10), uptimeSeconds: null }, { rebootRequired: false }, since), { restarted: true, confirmed: true });
|
||||
assert.deepEqual(restartOutcome({ lastSeenAt: iso(10), uptimeSeconds: null }, { rebootRequired: true }, since), { restarted: false, confirmed: true });
|
||||
assert.deepEqual(restartOutcome({ lastSeenAt: iso(10), uptimeSeconds: null }, null, since), { restarted: true, confirmed: false });
|
||||
});
|
||||
|
||||
test("the words for each case", () => {
|
||||
const yes = { restarted: true, confirmed: true };
|
||||
assert.equal(summarizeRestart("update", 12, 0, yes, false), "No updates are waiting now (12 updates installed); the server was restarted and is back.");
|
||||
assert.equal(summarizeRestart("restart", 12, 0, yes, false), "The server was restarted and is back.");
|
||||
assert.equal(summarizeRestart("restart", null, null, yes, true), "The server was restarted and is back, but says another restart is due.");
|
||||
assert.equal(summarizeRestart("restart", null, null, { restarted: false, confirmed: true }, true), "The playbook finished, but the server has not started again — check that the playbook really restarts it.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("a patch plan that restarts what needs it", () => {
|
||||
let aId = 0;
|
||||
let bId = 0;
|
||||
const plan = (extra: object = {}) => ({
|
||||
name: "Sunday night",
|
||||
enabled: true,
|
||||
days: JSON.stringify([0]),
|
||||
startTime: "02:00",
|
||||
timezone: "UTC",
|
||||
tags: "[]",
|
||||
serverIds: JSON.stringify([aId, bId]),
|
||||
integrationId: 9,
|
||||
projectId: 2,
|
||||
templateId: 7,
|
||||
templateName: "Apply updates",
|
||||
silenceDependents: false,
|
||||
windowMinutes: 90,
|
||||
...extra,
|
||||
});
|
||||
before(async () => {
|
||||
const [a] = await db.insert(servers).values({ name: "plan-a", apiTokenHash: hashToken("pa"), apiTokenPrefix: "pa", lastSeenAt: iso(0), uptimeSeconds: 86_400, pendingUpdates: updatesJson(3, false, 0) }).returning();
|
||||
const [b] = await db.insert(servers).values({ name: "plan-b", apiTokenHash: hashToken("pb"), apiTokenPrefix: "pb", lastSeenAt: iso(0), uptimeSeconds: 86_400, pendingUpdates: updatesJson(0, true, 0) }).returning();
|
||||
aId = a.id;
|
||||
bId = b.id;
|
||||
});
|
||||
beforeEach(async () => {
|
||||
resetFake();
|
||||
await db.update(servers).set({ lastSeenAt: iso(0), uptimeSeconds: 86_400, pendingUpdates: updatesJson(3, false, 0) }).where(eq(servers.id, aId));
|
||||
await db.update(servers).set({ lastSeenAt: iso(0), uptimeSeconds: 86_400, pendingUpdates: updatesJson(0, true, 0) }).where(eq(servers.id, bId));
|
||||
await db.delete(patchPlanRuns);
|
||||
await db.delete(patchPlans);
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
});
|
||||
|
||||
test("each server is updated with the restart playbook behind it, and a server that only needs a restart is restarted instead of left alone", async () => {
|
||||
const [p] = await db.insert(patchPlans).values(plan({ restartProjectId: 2, restartTemplateId: 8, restartTemplateName: "Restart server" })).returning();
|
||||
const run = await startPlanRun(p.id, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(at(1));
|
||||
const first = (await db.select().from(updateRuns).orderBy(updateRuns.id))[0];
|
||||
assert.equal(first.kind, "update");
|
||||
assert.equal(first.restartTemplateId, 8, "the update run has the restart playbook behind it");
|
||||
assert.equal(first.serverId, aId);
|
||||
|
||||
// plan-a finishes (no restart due)
|
||||
setTask(first.taskId!, { status: "success", end: iso(5) });
|
||||
await advanceUpdateRuns(at(6));
|
||||
await db.update(servers).set({ lastSeenAt: iso(8), uptimeSeconds: 86_400 + 480, pendingUpdates: updatesJson(0, false, 8) }).where(eq(servers.id, aId));
|
||||
await advanceUpdateRuns(at(9));
|
||||
await advancePlanRuns(at(10));
|
||||
|
||||
// plan-b has nothing waiting but needs a restart: restarted, not skipped
|
||||
const runs = await db.select().from(updateRuns).orderBy(updateRuns.id);
|
||||
assert.equal(runs.length, 2);
|
||||
assert.equal(runs[1].serverId, bId);
|
||||
assert.equal(runs[1].kind, "restart");
|
||||
assert.equal(runs[1].templateId, 8);
|
||||
const items = parseItems((await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, run.id)))[0].items);
|
||||
assert.equal(items[1].status, "running");
|
||||
assert.match(items[1].message!, /Nothing to update, but it needs a restart: the restart has started\./);
|
||||
});
|
||||
|
||||
test("without a restart playbook, a server with nothing waiting is left alone, as before", async () => {
|
||||
const [p] = await db.insert(patchPlans).values(plan()).returning();
|
||||
const run = await startPlanRun(p.id, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(at(1));
|
||||
const first = (await db.select().from(updateRuns))[0];
|
||||
assert.equal(first.restartTemplateId, null);
|
||||
setTask(first.taskId!, { status: "success", end: iso(5) });
|
||||
await advanceUpdateRuns(at(6));
|
||||
await db.update(servers).set({ lastSeenAt: iso(8), pendingUpdates: updatesJson(0, false, 8) }).where(eq(servers.id, aId));
|
||||
await advanceUpdateRuns(at(9));
|
||||
await advancePlanRuns(at(10));
|
||||
const items = parseItems((await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, run.id)))[0].items);
|
||||
assert.equal(items[1].status, "skipped");
|
||||
assert.equal(items[1].message, "No updates were waiting.");
|
||||
assert.equal((await db.select().from(updateRuns)).length, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
let serverId = 0;
|
||||
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "operator", name: "Olle", role: "operator" },
|
||||
{ oidcSub: "root", name: "Rut", role: "admin" },
|
||||
]);
|
||||
const [s] = await db.insert(servers).values({ name: "zoe", apiTokenHash: hashToken("tok3"), apiTokenPrefix: "t3", lastSeenAt: iso(0), uptimeSeconds: 100 }).returning();
|
||||
serverId = s.id;
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/update-runs", updateRunsRouter);
|
||||
a.use("/api/patch-plans", patchPlansRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
beforeEach(async () => {
|
||||
resetFake();
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
await db.delete(patchPlans);
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
const body = (extra: object = {}) => ({ serverId, integrationId: 9, projectId: 2, templateId: 7, limit: "zoe", windowMinutes: 60, ...extra });
|
||||
|
||||
test("an operator can start an update with a restart behind it, or a restart on its own, and the run says which", async () => {
|
||||
const res = await call("POST", "/update-runs", body({ restart: RESTART }));
|
||||
assert.equal(res.status, 201);
|
||||
const { run } = await res.json();
|
||||
assert.equal(run.kind, "update");
|
||||
assert.equal(run.restartTemplate, "Restart server");
|
||||
assert.equal(run.restarted, false);
|
||||
await db.delete(updateRuns);
|
||||
|
||||
const alone = await (await call("POST", "/update-runs", body({ kind: "restart", templateId: 8 }))).json();
|
||||
assert.equal(alone.run.kind, "restart");
|
||||
assert.equal(alone.run.template, "Restart server");
|
||||
assert.equal(alone.run.restartTemplate, null);
|
||||
await db.delete(updateRuns);
|
||||
|
||||
assert.equal((await call("POST", "/update-runs", body({ kind: "reboot" }))).status, 400);
|
||||
assert.equal((await call("POST", "/update-runs", body({ restart: { projectId: 2 } }))).status, 400);
|
||||
const missing = await call("POST", "/update-runs", body({ restart: { projectId: 2, templateId: 999 } }));
|
||||
assert.equal(missing.status, 404);
|
||||
assert.equal((await missing.json()).error, "restart_template_not_found");
|
||||
});
|
||||
|
||||
test("a patch plan keeps its restart playbook, all three parts or none, and an administrator's save is what changes it", async () => {
|
||||
const plan = (extra: object = {}) => ({ name: "Nights", days: [0], startTime: "02:00", timezone: "UTC", tags: ["lab"], integrationId: 9, projectId: 2, templateId: 7, templateName: "Apply updates", ...extra });
|
||||
who = "operator";
|
||||
assert.equal((await call("POST", "/patch-plans", plan())).status, 403);
|
||||
who = "root";
|
||||
const made = await call("POST", "/patch-plans", plan({ restartProjectId: 2, restartTemplateId: 8, restartTemplateName: "Restart server" }));
|
||||
assert.equal(made.status, 201);
|
||||
const { plan: saved } = await made.json();
|
||||
assert.deepEqual([saved.restartProjectId, saved.restartTemplateId, saved.restartTemplateName], [2, 8, "Restart server"]);
|
||||
|
||||
const half = await call("POST", "/patch-plans", plan({ name: "Half", restartTemplateId: 8 }));
|
||||
assert.equal(half.status, 400);
|
||||
assert.match((await half.json()).message, /needs its project, its template and its name — or none of them/);
|
||||
|
||||
const plain = await (await call("PUT", `/patch-plans/${saved.id}`, plan())).json();
|
||||
assert.deepEqual([plain.plan.restartProjectId, plain.plan.restartTemplateId, plain.plan.restartTemplateName], [null, null, null], "saving without one takes it away");
|
||||
who = "operator";
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user