An update run can name a second Semaphore playbook that restarts the server once the updates are in and it says a restart is due. The run keeps the maintenance window up, follows the restart, waits for the server to report again and checks from its uptime (or its restart flag clearing) that it did start again. A server that only needs a restart has a Restart button on the Updates page, and a patch plan can carry the restart playbook too. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
570 lines
31 KiB
TypeScript
570 lines
31 KiB
TypeScript
/**
|
|
* Restarting a server after its updates: a second playbook, run when the server says a restart is due; the quiet kept up for it; the server waited for and checked
|
|
* to have started again (from its uptime, or its "restart needed" flag clearing); a restart on its own; and a patch plan that restarts what needs it.
|
|
* Semaphore is a stand-in that answers as told.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after, beforeEach } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { AddressInfo } from "node:net";
|
|
import type { Server } from "node:http";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-restart-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const express = (await import("express")).default;
|
|
const { db } = await import("../src/db/client.js");
|
|
const { users, servers, updateRuns, auditLog, maintenanceWindows, patchPlans, patchPlanRuns } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const { startUpdateRun, advanceUpdateRuns, setSemaphoreForTests, startedSince, restartOutcome, summarizeRestart, UpdateRunError, HOST_WAIT_MS, RESTART_WINDOW_MINUTES, ACTIVE_STATUSES } = await import("../src/services/updateRuns.js");
|
|
const { startPlanRun, advancePlanRuns, parseItems } = await import("../src/services/patchPlans.js");
|
|
const { listActiveWindows } = await import("../src/services/maintenance.js");
|
|
const { getServerActivity } = await import("../src/services/serverActivity.js");
|
|
const { updateRunsRouter } = await import("../src/routes/updateRuns.js");
|
|
const { patchPlansRouter } = await import("../src/routes/patchPlans.js");
|
|
const { hashToken } = await import("../src/services/tokens.js");
|
|
const { eq } = await import("drizzle-orm");
|
|
|
|
await runMigrations();
|
|
|
|
// ── a Semaphore that answers as told: two templates (updates, restart), and a task for every run ──
|
|
interface Task {
|
|
id: number;
|
|
status: string;
|
|
created: string | null;
|
|
start: string | null;
|
|
end: string | null;
|
|
}
|
|
const fake = { runs: [] as { projectId: number; templateId: number; limit?: string; message?: string }[], tasks: new Map<number, Task>(), next: 100, refuseTemplate: null as number | null };
|
|
const resetFake = () => {
|
|
fake.runs = [];
|
|
fake.tasks = new Map();
|
|
fake.next = 100;
|
|
fake.refuseTemplate = null;
|
|
};
|
|
const setTask = (id: number, patch: Partial<Task>) => fake.tasks.set(id, { ...fake.tasks.get(id)!, ...patch });
|
|
setSemaphoreForTests(async (integrationId) =>
|
|
integrationId === 9
|
|
? {
|
|
ping: async () => ({ ok: true }),
|
|
listTemplatesWithStatus: async () => [],
|
|
checkTemplates: async () => ({
|
|
templates: [
|
|
{ id: 7, projectId: 2, projectName: "Homelab", name: "Apply updates", playbook: "updates.yml", lastTask: null },
|
|
{ id: 8, projectId: 2, projectName: "Homelab", name: "Restart server", playbook: "restart.yml", lastTask: null },
|
|
],
|
|
failedProjectIds: [],
|
|
}),
|
|
runTemplate: async (projectId, templateId, options) => {
|
|
if (fake.refuseTemplate === templateId) throw new Error("Semaphore is having a moment");
|
|
fake.runs.push({ projectId, templateId, limit: options?.limit, message: options?.message });
|
|
const task = { id: fake.next++, status: "waiting", created: null, start: null, end: null };
|
|
fake.tasks.set(task.id, task);
|
|
return { ...task } as never;
|
|
},
|
|
getTask: async (_p, id) => ({ ...fake.tasks.get(id)! }) as never,
|
|
}
|
|
: null,
|
|
);
|
|
|
|
const T0 = new Date("2026-10-17T12:00:00.000Z");
|
|
const at = (minutes: number) => new Date(T0.getTime() + minutes * 60_000);
|
|
const iso = (minutes: number) => at(minutes).toISOString();
|
|
const RESTART = { projectId: 2, templateId: 8 };
|
|
const input = (serverId: number, extra: object = {}) => ({ serverId, integrationId: 9, projectId: 2, templateId: 7, limit: "elsa", windowMinutes: 90, actor: "Olle", ...extra });
|
|
const runRow = async (id: number) => (await db.select().from(updateRuns).where(eq(updateRuns.id, id)))[0];
|
|
const updatesJson = (count: number, reboot: boolean, minutes: number) => JSON.stringify({ manager: "apt", count, security: 0, rebootRequired: reboot, reportedAt: iso(minutes) });
|
|
|
|
describe("restarting after the updates", () => {
|
|
let serverId = 0;
|
|
before(async () => {
|
|
await db.insert(users).values([{ oidcSub: "admin", name: "Ada", role: "admin" }]);
|
|
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: iso(0), uptimeSeconds: 86_400 * 30, pendingUpdates: updatesJson(12, false, 0) }).returning();
|
|
serverId = s.id;
|
|
});
|
|
beforeEach(async () => {
|
|
resetFake();
|
|
await db.delete(updateRuns);
|
|
await db.delete(maintenanceWindows);
|
|
await db.update(servers).set({ lastSeenAt: iso(0), uptimeSeconds: 86_400 * 30, pendingUpdates: updatesJson(12, false, 0) }).where(eq(servers.id, serverId));
|
|
});
|
|
after(() => {
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows keeps the database file locked until the process exits.
|
|
}
|
|
});
|
|
|
|
/** Reports as the agent would: when, what is waiting, whether a restart is due, and how long it has been up. */
|
|
const report = (minutes: number, count: number, reboot: boolean, uptimeSeconds: number | null) =>
|
|
db.update(servers).set({ lastSeenAt: iso(minutes), uptimeSeconds, pendingUpdates: updatesJson(count, reboot, minutes) }).where(eq(servers.id, serverId));
|
|
|
|
/** Runs an update with a restart playbook to the point where the server says a restart is due. */
|
|
async function toRestart(start = 0) {
|
|
const run = await startUpdateRun(input(serverId, { restart: RESTART }), at(start));
|
|
setTask(run.taskId!, { status: "success", end: iso(start + 8) });
|
|
await advanceUpdateRuns(at(start + 9));
|
|
await report(start + 14, 2, true, 86_400 * 30 + 840);
|
|
await advanceUpdateRuns(at(start + 15));
|
|
return run;
|
|
}
|
|
|
|
test("an update run remembers the restart playbook, says so in its history, and refuses one that isn't in Semaphore", async () => {
|
|
const run = await startUpdateRun(input(serverId, { restart: RESTART }), T0);
|
|
assert.equal(run.kind, "update");
|
|
assert.deepEqual([run.restartProjectId, run.restartTemplateId, run.restartTemplateName], [2, 8, "Restart server"]);
|
|
assert.equal(fake.runs.length, 1, "only the update playbook is started now");
|
|
const events = await getServerActivity(serverId, at(1));
|
|
assert.ok(events.some((e) => e.title === "Update run started" && /then a restart with Restart server if one is due/.test(e.detail ?? "")));
|
|
|
|
await db.delete(updateRuns);
|
|
await db.delete(maintenanceWindows);
|
|
fake.runs = [];
|
|
await assert.rejects(startUpdateRun(input(serverId, { restart: { projectId: 2, templateId: 999 } }), T0), (e: unknown) => e instanceof UpdateRunError && e.status === 404 && e.code === "restart_template_not_found");
|
|
assert.equal(fake.runs.length, 0, "nothing was started");
|
|
assert.equal((await listActiveWindows(T0)).length, 0, "and no quiet was left behind");
|
|
});
|
|
|
|
test("when the server says a restart is due, the restart playbook runs, is followed, and the server is checked to have started again", async () => {
|
|
const run = await toRestart();
|
|
let r = await runRow(run.id);
|
|
assert.equal(r.status, "restarting");
|
|
assert.equal(r.updatesAfter, 2);
|
|
assert.equal(r.restartStartedAt, iso(15));
|
|
assert.deepEqual(fake.runs.map((x) => [x.templateId, x.limit]), [[7, "elsa"], [8, "elsa"]], "the same server, and only that one");
|
|
assert.match(fake.runs[1].message!, /Restart of elsa after update run #\d+/);
|
|
assert.match(r.message!, /2 updates still waiting \(10 updates installed\)\. The server says a restart is due — restarting it with "Restart server"/);
|
|
assert.ok(ACTIVE_STATUSES.includes(r.status as never), "it counts as under way");
|
|
assert.ok((await db.select().from(auditLog)).some((a) => a.category === "update_run" && a.action === "restart"));
|
|
|
|
// the quiet is kept up for the restart (it was the update's 90 minutes; the restart's own is shorter, so it stays at 90)
|
|
const windows = await listActiveWindows(at(16));
|
|
assert.equal(windows.length, 1);
|
|
assert.equal(windows[0].id, r.windowId);
|
|
|
|
setTask(r.restartTaskId!, { status: "running" });
|
|
await advanceUpdateRuns(at(17));
|
|
r = await runRow(run.id);
|
|
assert.equal(r.status, "restarting");
|
|
assert.equal(r.restartTaskStatus, "running");
|
|
assert.equal(r.taskStatus, "success", "the update's own status is left as it was");
|
|
|
|
setTask(r.restartTaskId!, { status: "success", end: iso(19) });
|
|
await advanceUpdateRuns(at(20));
|
|
r = await runRow(run.id);
|
|
assert.equal(r.status, "waiting_for_restart");
|
|
assert.equal(r.restartEndedAt, iso(19));
|
|
assert.match(r.message!, /Waiting for the server to come back/);
|
|
|
|
await advanceUpdateRuns(at(22));
|
|
assert.equal((await runRow(run.id)).status, "waiting_for_restart", "the report from before the restart ended doesn't count");
|
|
|
|
await report(23, 2, false, 150); // up for two and a half minutes: it started at 20:30, after the restart began
|
|
await advanceUpdateRuns(at(24));
|
|
r = await runRow(run.id);
|
|
assert.equal(r.status, "done");
|
|
assert.equal(r.restarted, true);
|
|
assert.equal(r.rebootRequired, false);
|
|
assert.equal(r.message, "2 updates still waiting (10 updates installed); the server was restarted and is back.");
|
|
assert.equal((await listActiveWindows(at(25))).length, 0, "the window it made is over");
|
|
});
|
|
|
|
test("a server that says no restart is due is not restarted", async () => {
|
|
const run = await startUpdateRun(input(serverId, { restart: RESTART }), T0);
|
|
setTask(run.taskId!, { status: "success", end: iso(8) });
|
|
await advanceUpdateRuns(at(9));
|
|
await report(14, 0, false, 600);
|
|
await advanceUpdateRuns(at(15));
|
|
const r = await runRow(run.id);
|
|
assert.equal(r.status, "done");
|
|
assert.equal(r.restarted, false);
|
|
assert.equal(r.message, "No updates are waiting now (12 updates installed); no restart is needed.");
|
|
assert.equal(fake.runs.length, 1, "the restart playbook was never run");
|
|
});
|
|
|
|
test("without a restart playbook a restart that is due is only said to be, as before", async () => {
|
|
const run = await startUpdateRun(input(serverId), T0);
|
|
setTask(run.taskId!, { status: "success", end: iso(8) });
|
|
await advanceUpdateRuns(at(9));
|
|
await report(14, 2, true, 840);
|
|
await advanceUpdateRuns(at(15));
|
|
const r = await runRow(run.id);
|
|
assert.equal(r.status, "done");
|
|
assert.equal(r.message, "2 updates still waiting (10 updates installed); a restart is still needed.");
|
|
assert.equal(fake.runs.length, 1);
|
|
});
|
|
|
|
test("if the restart playbook fails, the run says the updates are in and the server still needs a restart; the quiet ends", async () => {
|
|
const run = await toRestart();
|
|
const { restartTaskId } = await runRow(run.id);
|
|
setTask(restartTaskId!, { status: "error" });
|
|
await advanceUpdateRuns(at(18));
|
|
const r = await runRow(run.id);
|
|
assert.equal(r.status, "failed");
|
|
assert.match(r.message!, /The updates are in, but the restart didn't happen\. The playbook failed in Semaphore \(run #\d+\)\. The server still needs a restart/);
|
|
assert.equal((await listActiveWindows(at(19))).length, 0);
|
|
});
|
|
|
|
test("if the restart playbook can't be started, the run ends saying so, with the updates counted", async () => {
|
|
fake.refuseTemplate = 8;
|
|
const run = await startUpdateRun(input(serverId, { restart: RESTART }), T0);
|
|
setTask(run.taskId!, { status: "success", end: iso(8) });
|
|
await advanceUpdateRuns(at(9));
|
|
await report(14, 2, true, 840);
|
|
await advanceUpdateRuns(at(15));
|
|
const r = await runRow(run.id);
|
|
assert.equal(r.status, "failed");
|
|
assert.equal(r.message, "2 updates still waiting (10 updates installed), but the restart could not be started: Semaphore is having a moment. The server still needs a restart.");
|
|
assert.equal(r.rebootRequired, true);
|
|
assert.equal((await listActiveWindows(at(16))).length, 0);
|
|
});
|
|
|
|
test("a playbook that finished without the server restarting is a failure, from its uptime — and a restart flag that never clears is too", async () => {
|
|
// reported again, but it has been up since long before the restart began
|
|
let run = await toRestart();
|
|
setTask((await runRow(run.id)).restartTaskId!, { status: "success", end: iso(19) });
|
|
await advanceUpdateRuns(at(20));
|
|
await report(23, 2, false, 86_400 * 30 + 1380);
|
|
await advanceUpdateRuns(at(24));
|
|
let r = await runRow(run.id);
|
|
assert.equal(r.status, "failed");
|
|
assert.equal(r.restarted, false);
|
|
assert.equal(r.message, "2 updates still waiting (10 updates installed); the restart playbook finished, but the server has not started again — check that the playbook really restarts it.");
|
|
|
|
// no uptime in its reports: the restart flag is what there is
|
|
await db.delete(updateRuns);
|
|
await db.delete(maintenanceWindows);
|
|
run = await toRestart(100);
|
|
setTask((await runRow(run.id)).restartTaskId!, { status: "success", end: iso(119) });
|
|
await advanceUpdateRuns(at(120));
|
|
await report(123, 2, true, null);
|
|
await advanceUpdateRuns(at(124));
|
|
r = await runRow(run.id);
|
|
assert.equal(r.status, "failed", "still says a restart is due");
|
|
assert.match(r.message!, /has not started again/);
|
|
});
|
|
|
|
test("with no uptime to go by, the flag clearing is enough; with neither, it is taken to have restarted and says it couldn't be confirmed", async () => {
|
|
let run = await toRestart();
|
|
setTask((await runRow(run.id)).restartTaskId!, { status: "success", end: iso(19) });
|
|
await advanceUpdateRuns(at(20));
|
|
await report(23, 2, false, null);
|
|
await advanceUpdateRuns(at(24));
|
|
let r = await runRow(run.id);
|
|
assert.equal(r.status, "done");
|
|
assert.equal(r.message, "2 updates still waiting (10 updates installed); the server was restarted and is back.");
|
|
|
|
await db.delete(updateRuns);
|
|
await db.delete(maintenanceWindows);
|
|
run = await toRestart(100);
|
|
setTask((await runRow(run.id)).restartTaskId!, { status: "success", end: iso(119) });
|
|
await advanceUpdateRuns(at(120));
|
|
await db.update(servers).set({ lastSeenAt: iso(123), uptimeSeconds: null, pendingUpdates: null }).where(eq(servers.id, serverId));
|
|
await advanceUpdateRuns(at(124));
|
|
r = await runRow(run.id);
|
|
assert.equal(r.status, "done");
|
|
assert.match(r.message!, /was restarted and is back \(its uptime isn't reported, so that couldn't be confirmed\)\.$/);
|
|
});
|
|
|
|
test("a restarted server that says another restart is due is done, with that said", async () => {
|
|
const run = await toRestart();
|
|
setTask((await runRow(run.id)).restartTaskId!, { status: "success", end: iso(19) });
|
|
await advanceUpdateRuns(at(20));
|
|
await report(23, 0, true, 150);
|
|
await advanceUpdateRuns(at(24));
|
|
const r = await runRow(run.id);
|
|
assert.equal(r.status, "done");
|
|
assert.equal(r.rebootRequired, true);
|
|
assert.match(r.message!, /the server was restarted and is back, but says another restart is due\.$/);
|
|
});
|
|
|
|
test("a server that doesn't come back from its restart is given up on, and the quiet ends", async () => {
|
|
const run = await toRestart();
|
|
setTask((await runRow(run.id)).restartTaskId!, { status: "success", end: iso(19) });
|
|
await advanceUpdateRuns(at(20));
|
|
await advanceUpdateRuns(at(19 + HOST_WAIT_MS / 60_000 + 1));
|
|
const r = await runRow(run.id);
|
|
assert.equal(r.status, "failed");
|
|
assert.match(r.message!, /The restart playbook finished, but the server hasn't reported again in 30 minutes/);
|
|
assert.equal((await listActiveWindows(at(60))).length, 0);
|
|
});
|
|
|
|
test("the quiet is made again if it ended before the restart began, and is not shortened if it is longer than the restart needs", async () => {
|
|
// a window longer than the restart needs (somebody made it longer while the update ran) is not cut short
|
|
const run = await startUpdateRun(input(serverId, { restart: RESTART }), T0);
|
|
await db.update(maintenanceWindows).set({ endsAt: iso(600) }).where(eq(maintenanceWindows.id, run.windowId!));
|
|
setTask(run.taskId!, { status: "success", end: iso(8) });
|
|
await advanceUpdateRuns(at(9));
|
|
await report(14, 2, true, 840);
|
|
await advanceUpdateRuns(at(15));
|
|
assert.equal((await runRow(run.id)).status, "restarting");
|
|
assert.equal((await listActiveWindows(at(16)))[0].endsAt, iso(600), "left as long as it was");
|
|
await db.delete(updateRuns);
|
|
await db.delete(maintenanceWindows);
|
|
|
|
// one that has already run out is made again, for the restart
|
|
const second = await startUpdateRun(input(serverId, { restart: RESTART, windowMinutes: 15 }), at(200));
|
|
setTask(second.taskId!, { status: "success", end: iso(208) });
|
|
await advanceUpdateRuns(at(209));
|
|
await report(214, 2, true, 600);
|
|
await advanceUpdateRuns(at(216)); // the 15 minutes ran out at 215
|
|
const r = await runRow(second.id);
|
|
assert.equal(r.status, "restarting");
|
|
const windows = await listActiveWindows(at(217));
|
|
assert.equal(windows.length, 1, "quiet again for the restart");
|
|
assert.equal(Date.parse(windows[0].endsAt) - at(216).getTime(), RESTART_WINDOW_MINUTES * 60_000);
|
|
assert.equal(r.windowId, windows[0].id);
|
|
});
|
|
});
|
|
|
|
describe("restarting a server on its own", () => {
|
|
let serverId = 0;
|
|
before(async () => {
|
|
const [s] = await db.insert(servers).values({ name: "anna", apiTokenHash: hashToken("tok2"), apiTokenPrefix: "t2", lastSeenAt: iso(0), uptimeSeconds: 86_400 * 10, pendingUpdates: updatesJson(0, true, 0) }).returning();
|
|
serverId = s.id;
|
|
});
|
|
beforeEach(async () => {
|
|
resetFake();
|
|
await db.delete(updateRuns);
|
|
await db.delete(maintenanceWindows);
|
|
await db.update(servers).set({ lastSeenAt: iso(0), uptimeSeconds: 86_400 * 10, pendingUpdates: updatesJson(0, true, 0) }).where(eq(servers.id, serverId));
|
|
});
|
|
|
|
test("a restart run runs the playbook it was given, follows it, and checks the server started again", async () => {
|
|
const run = await startUpdateRun(input(serverId, { kind: "restart", templateId: 8, limit: "anna", restart: RESTART }), T0);
|
|
assert.equal(run.kind, "restart");
|
|
assert.equal(run.templateName, "Restart server");
|
|
assert.equal(run.restartTemplateId, null, "a restart doesn't restart again");
|
|
assert.deepEqual(fake.runs.map((x) => [x.templateId, x.limit]), [[8, "anna"]]);
|
|
assert.match(fake.runs[0].message!, /^Restart for anna, started in Homelab Manager by Olle/);
|
|
const windows = await listActiveWindows(T0);
|
|
assert.equal(windows[0].reason, "Restart: Restart server");
|
|
|
|
setTask(run.taskId!, { status: "success", end: iso(4) });
|
|
await advanceUpdateRuns(at(5));
|
|
let r = await runRow(run.id);
|
|
assert.equal(r.status, "waiting_for_host");
|
|
assert.match(r.message!, /The restart playbook finished\. Waiting for the server to come back/);
|
|
|
|
await db.update(servers).set({ lastSeenAt: iso(9), uptimeSeconds: 200, pendingUpdates: updatesJson(0, false, 9) }).where(eq(servers.id, serverId));
|
|
await advanceUpdateRuns(at(10));
|
|
r = await runRow(run.id);
|
|
assert.equal(r.status, "done");
|
|
assert.equal(r.restarted, true);
|
|
assert.equal(r.message, "The server was restarted and is back.");
|
|
assert.equal((await listActiveWindows(at(11))).length, 0);
|
|
});
|
|
|
|
test("a restart that did not happen fails, in its own words", async () => {
|
|
const run = await startUpdateRun(input(serverId, { kind: "restart", templateId: 8, limit: "anna" }), T0);
|
|
setTask(run.taskId!, { status: "success", end: iso(4) });
|
|
await advanceUpdateRuns(at(5));
|
|
await db.update(servers).set({ lastSeenAt: iso(9), uptimeSeconds: 86_400 * 10 + 540, pendingUpdates: updatesJson(0, true, 9) }).where(eq(servers.id, serverId));
|
|
await advanceUpdateRuns(at(10));
|
|
const r = await runRow(run.id);
|
|
assert.equal(r.status, "failed");
|
|
assert.equal(r.message, "The playbook finished, but the server has not started again — check that the playbook really restarts it.");
|
|
});
|
|
|
|
test("it is in the server's history as a restart", async () => {
|
|
await startUpdateRun(input(serverId, { kind: "restart", templateId: 8, limit: "anna" }), T0);
|
|
const events = await getServerActivity(serverId, at(1));
|
|
assert.ok(events.some((e) => e.title === "Restart started"));
|
|
});
|
|
});
|
|
|
|
describe("working out whether a server has started again", () => {
|
|
const since = T0.getTime();
|
|
test("from its uptime: the report's time minus the uptime is when it started", () => {
|
|
assert.equal(startedSince({ lastSeenAt: iso(10), uptimeSeconds: 300 }, since), true, "started at 12:05");
|
|
assert.equal(startedSince({ lastSeenAt: iso(10), uptimeSeconds: 86_400 }, since), false);
|
|
assert.equal(startedSince({ lastSeenAt: iso(10), uptimeSeconds: 10 * 60 + 60 }, since), true, "a minute before the restart began still counts");
|
|
assert.equal(startedSince({ lastSeenAt: iso(10), uptimeSeconds: 10 * 60 + 600 }, since), false, "ten minutes before does not");
|
|
assert.equal(startedSince({ lastSeenAt: iso(10), uptimeSeconds: null }, since), null);
|
|
assert.equal(startedSince({ lastSeenAt: null, uptimeSeconds: 100 }, since), null);
|
|
});
|
|
|
|
test("the uptime is believed over the flag; the flag over nothing", () => {
|
|
assert.deepEqual(restartOutcome({ lastSeenAt: iso(10), uptimeSeconds: 300 }, { rebootRequired: true }, since), { restarted: true, confirmed: true });
|
|
assert.deepEqual(restartOutcome({ lastSeenAt: iso(10), uptimeSeconds: 86_400 }, { rebootRequired: false }, since), { restarted: false, confirmed: true });
|
|
assert.deepEqual(restartOutcome({ lastSeenAt: iso(10), uptimeSeconds: null }, { rebootRequired: false }, since), { restarted: true, confirmed: true });
|
|
assert.deepEqual(restartOutcome({ lastSeenAt: iso(10), uptimeSeconds: null }, { rebootRequired: true }, since), { restarted: false, confirmed: true });
|
|
assert.deepEqual(restartOutcome({ lastSeenAt: iso(10), uptimeSeconds: null }, null, since), { restarted: true, confirmed: false });
|
|
});
|
|
|
|
test("the words for each case", () => {
|
|
const yes = { restarted: true, confirmed: true };
|
|
assert.equal(summarizeRestart("update", 12, 0, yes, false), "No updates are waiting now (12 updates installed); the server was restarted and is back.");
|
|
assert.equal(summarizeRestart("restart", 12, 0, yes, false), "The server was restarted and is back.");
|
|
assert.equal(summarizeRestart("restart", null, null, yes, true), "The server was restarted and is back, but says another restart is due.");
|
|
assert.equal(summarizeRestart("restart", null, null, { restarted: false, confirmed: true }, true), "The playbook finished, but the server has not started again — check that the playbook really restarts it.");
|
|
});
|
|
});
|
|
|
|
describe("a patch plan that restarts what needs it", () => {
|
|
let aId = 0;
|
|
let bId = 0;
|
|
const plan = (extra: object = {}) => ({
|
|
name: "Sunday night",
|
|
enabled: true,
|
|
days: JSON.stringify([0]),
|
|
startTime: "02:00",
|
|
timezone: "UTC",
|
|
tags: "[]",
|
|
serverIds: JSON.stringify([aId, bId]),
|
|
integrationId: 9,
|
|
projectId: 2,
|
|
templateId: 7,
|
|
templateName: "Apply updates",
|
|
silenceDependents: false,
|
|
windowMinutes: 90,
|
|
...extra,
|
|
});
|
|
before(async () => {
|
|
const [a] = await db.insert(servers).values({ name: "plan-a", apiTokenHash: hashToken("pa"), apiTokenPrefix: "pa", lastSeenAt: iso(0), uptimeSeconds: 86_400, pendingUpdates: updatesJson(3, false, 0) }).returning();
|
|
const [b] = await db.insert(servers).values({ name: "plan-b", apiTokenHash: hashToken("pb"), apiTokenPrefix: "pb", lastSeenAt: iso(0), uptimeSeconds: 86_400, pendingUpdates: updatesJson(0, true, 0) }).returning();
|
|
aId = a.id;
|
|
bId = b.id;
|
|
});
|
|
beforeEach(async () => {
|
|
resetFake();
|
|
await db.update(servers).set({ lastSeenAt: iso(0), uptimeSeconds: 86_400, pendingUpdates: updatesJson(3, false, 0) }).where(eq(servers.id, aId));
|
|
await db.update(servers).set({ lastSeenAt: iso(0), uptimeSeconds: 86_400, pendingUpdates: updatesJson(0, true, 0) }).where(eq(servers.id, bId));
|
|
await db.delete(patchPlanRuns);
|
|
await db.delete(patchPlans);
|
|
await db.delete(updateRuns);
|
|
await db.delete(maintenanceWindows);
|
|
});
|
|
|
|
test("each server is updated with the restart playbook behind it, and a server that only needs a restart is restarted instead of left alone", async () => {
|
|
const [p] = await db.insert(patchPlans).values(plan({ restartProjectId: 2, restartTemplateId: 8, restartTemplateName: "Restart server" })).returning();
|
|
const run = await startPlanRun(p.id, { trigger: "manual", actor: "Olle", now: T0 });
|
|
await advancePlanRuns(at(1));
|
|
const first = (await db.select().from(updateRuns).orderBy(updateRuns.id))[0];
|
|
assert.equal(first.kind, "update");
|
|
assert.equal(first.restartTemplateId, 8, "the update run has the restart playbook behind it");
|
|
assert.equal(first.serverId, aId);
|
|
|
|
// plan-a finishes (no restart due)
|
|
setTask(first.taskId!, { status: "success", end: iso(5) });
|
|
await advanceUpdateRuns(at(6));
|
|
await db.update(servers).set({ lastSeenAt: iso(8), uptimeSeconds: 86_400 + 480, pendingUpdates: updatesJson(0, false, 8) }).where(eq(servers.id, aId));
|
|
await advanceUpdateRuns(at(9));
|
|
await advancePlanRuns(at(10));
|
|
|
|
// plan-b has nothing waiting but needs a restart: restarted, not skipped
|
|
const runs = await db.select().from(updateRuns).orderBy(updateRuns.id);
|
|
assert.equal(runs.length, 2);
|
|
assert.equal(runs[1].serverId, bId);
|
|
assert.equal(runs[1].kind, "restart");
|
|
assert.equal(runs[1].templateId, 8);
|
|
const items = parseItems((await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, run.id)))[0].items);
|
|
assert.equal(items[1].status, "running");
|
|
assert.match(items[1].message!, /Nothing to update, but it needs a restart: the restart has started\./);
|
|
});
|
|
|
|
test("without a restart playbook, a server with nothing waiting is left alone, as before", async () => {
|
|
const [p] = await db.insert(patchPlans).values(plan()).returning();
|
|
const run = await startPlanRun(p.id, { trigger: "manual", actor: "Olle", now: T0 });
|
|
await advancePlanRuns(at(1));
|
|
const first = (await db.select().from(updateRuns))[0];
|
|
assert.equal(first.restartTemplateId, null);
|
|
setTask(first.taskId!, { status: "success", end: iso(5) });
|
|
await advanceUpdateRuns(at(6));
|
|
await db.update(servers).set({ lastSeenAt: iso(8), pendingUpdates: updatesJson(0, false, 8) }).where(eq(servers.id, aId));
|
|
await advanceUpdateRuns(at(9));
|
|
await advancePlanRuns(at(10));
|
|
const items = parseItems((await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, run.id)))[0].items);
|
|
assert.equal(items[1].status, "skipped");
|
|
assert.equal(items[1].message, "No updates were waiting.");
|
|
assert.equal((await db.select().from(updateRuns)).length, 1);
|
|
});
|
|
});
|
|
|
|
describe("the routes", () => {
|
|
let app: Server;
|
|
let base = "";
|
|
let who = "operator";
|
|
let serverId = 0;
|
|
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
|
|
|
before(async () => {
|
|
await db.insert(users).values([
|
|
{ oidcSub: "operator", name: "Olle", role: "operator" },
|
|
{ oidcSub: "root", name: "Rut", role: "admin" },
|
|
]);
|
|
const [s] = await db.insert(servers).values({ name: "zoe", apiTokenHash: hashToken("tok3"), apiTokenPrefix: "t3", lastSeenAt: iso(0), uptimeSeconds: 100 }).returning();
|
|
serverId = s.id;
|
|
const a = express();
|
|
a.use(express.json());
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
|
next();
|
|
});
|
|
a.use("/api/update-runs", updateRunsRouter);
|
|
a.use("/api/patch-plans", patchPlansRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
|
});
|
|
beforeEach(async () => {
|
|
resetFake();
|
|
await db.delete(updateRuns);
|
|
await db.delete(maintenanceWindows);
|
|
await db.delete(patchPlans);
|
|
});
|
|
after(() => app.close());
|
|
|
|
const body = (extra: object = {}) => ({ serverId, integrationId: 9, projectId: 2, templateId: 7, limit: "zoe", windowMinutes: 60, ...extra });
|
|
|
|
test("an operator can start an update with a restart behind it, or a restart on its own, and the run says which", async () => {
|
|
const res = await call("POST", "/update-runs", body({ restart: RESTART }));
|
|
assert.equal(res.status, 201);
|
|
const { run } = await res.json();
|
|
assert.equal(run.kind, "update");
|
|
assert.equal(run.restartTemplate, "Restart server");
|
|
assert.equal(run.restarted, false);
|
|
await db.delete(updateRuns);
|
|
|
|
const alone = await (await call("POST", "/update-runs", body({ kind: "restart", templateId: 8 }))).json();
|
|
assert.equal(alone.run.kind, "restart");
|
|
assert.equal(alone.run.template, "Restart server");
|
|
assert.equal(alone.run.restartTemplate, null);
|
|
await db.delete(updateRuns);
|
|
|
|
assert.equal((await call("POST", "/update-runs", body({ kind: "reboot" }))).status, 400);
|
|
assert.equal((await call("POST", "/update-runs", body({ restart: { projectId: 2 } }))).status, 400);
|
|
const missing = await call("POST", "/update-runs", body({ restart: { projectId: 2, templateId: 999 } }));
|
|
assert.equal(missing.status, 404);
|
|
assert.equal((await missing.json()).error, "restart_template_not_found");
|
|
});
|
|
|
|
test("a patch plan keeps its restart playbook, all three parts or none, and an administrator's save is what changes it", async () => {
|
|
const plan = (extra: object = {}) => ({ name: "Nights", days: [0], startTime: "02:00", timezone: "UTC", tags: ["lab"], integrationId: 9, projectId: 2, templateId: 7, templateName: "Apply updates", ...extra });
|
|
who = "operator";
|
|
assert.equal((await call("POST", "/patch-plans", plan())).status, 403);
|
|
who = "root";
|
|
const made = await call("POST", "/patch-plans", plan({ restartProjectId: 2, restartTemplateId: 8, restartTemplateName: "Restart server" }));
|
|
assert.equal(made.status, 201);
|
|
const { plan: saved } = await made.json();
|
|
assert.deepEqual([saved.restartProjectId, saved.restartTemplateId, saved.restartTemplateName], [2, 8, "Restart server"]);
|
|
|
|
const half = await call("POST", "/patch-plans", plan({ name: "Half", restartTemplateId: 8 }));
|
|
assert.equal(half.status, 400);
|
|
assert.match((await half.json()).message, /needs its project, its template and its name — or none of them/);
|
|
|
|
const plain = await (await call("PUT", `/patch-plans/${saved.id}`, plan())).json();
|
|
assert.deepEqual([plain.plan.restartProjectId, plain.plan.restartTemplateId, plain.plan.restartTemplateName], [null, null, null], "saving without one takes it away");
|
|
who = "operator";
|
|
});
|
|
});
|