Stale-backup alerts, per-server alert limits, and update runs from the Updates page
- Backups: a guest whose newest backup is older than its limit is an alert (backup kind), in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds) - Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent, forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the global ones were; saving clears the alerts cache - Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and reports what is left and whether a restart is due; poller survives restarts - Notification toggle for update-run results; tests and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5dc6cea69c
commit
7a5b623444
42 files changed
+7561
-57
No files matched your search
@@ -0,0 +1,271 @@
|
||||
/**
|
||||
* An update run: apply a server's waiting updates in one go, and see it through. It puts the server under maintenance (so nothing alerts
|
||||
* while it is being worked on and restarted), starts a Semaphore playbook limited to that server, follows the run until it is over, waits
|
||||
* for the server's agent to report again, then says how many updates are still waiting and whether a restart is due, and ends the
|
||||
* maintenance window it made. Every step is written down on the run, which survives a restart of this app — a poller picks up where it was.
|
||||
*
|
||||
* What it does not do: it does not restart the server (the playbook does, if it is written to), and it cannot stop a run in Semaphore —
|
||||
* cancelling here only stops watching and ends the window.
|
||||
*/
|
||||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { maintenanceWindows, servers, updateRuns } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createSemaphoreAdapter, type SemaphoreAdapter, type SemaphoreTask } from "../integrations/semaphore/adapter.js";
|
||||
import { invalidateAlertsCache } from "./alerts.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { startOrExtendWindow } from "./maintenance.js";
|
||||
import { notifyUpdateRunFinished } from "./notify.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
|
||||
export type UpdateRunRow = typeof updateRuns.$inferSelect;
|
||||
export type UpdateRunStatus = "running" | "waiting_for_host" | "done" | "failed" | "cancelled";
|
||||
export const ACTIVE_STATUSES: UpdateRunStatus[] = ["running", "waiting_for_host"];
|
||||
|
||||
/** How long a run may take from start to finish before it is given up on. */
|
||||
export const RUN_LIMIT_MS = 3 * 3600_000;
|
||||
/** After the playbook is over, how long the server has to report again (it may be restarting). */
|
||||
export const HOST_WAIT_MS = 30 * 60_000;
|
||||
/** Consecutive polls Semaphore may fail to answer before the run is given up on (at the poller's pace, about five minutes). */
|
||||
export const MAX_POLL_ERRORS = 10;
|
||||
export const POLL_INTERVAL_MS = 20_000;
|
||||
|
||||
const FINISHED_TASK = new Set<SemaphoreTask["status"]>(["success", "error", "stopped", "rejected"]);
|
||||
|
||||
// ─── Semaphore, replaceable for the tests ───────────────────────────────────
|
||||
|
||||
let adapterOverride: ((integrationId: number) => Promise<SemaphoreAdapter | null>) | null = null;
|
||||
export function setSemaphoreForTests(factory: ((integrationId: number) => Promise<SemaphoreAdapter | null>) | null): void {
|
||||
adapterOverride = factory;
|
||||
}
|
||||
|
||||
async function semaphoreFor(integrationId: number): Promise<SemaphoreAdapter | null> {
|
||||
if (adapterOverride) return adapterOverride(integrationId);
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "semaphore" || !loaded.integration.enabled) return null;
|
||||
return createSemaphoreAdapter(loaded.config as any);
|
||||
}
|
||||
|
||||
// ─── Starting ───────────────────────────────────────────────────────────────
|
||||
|
||||
export class UpdateRunError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly code: string,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export interface StartInput {
|
||||
serverId: number;
|
||||
integrationId: number;
|
||||
projectId: number;
|
||||
templateId: number;
|
||||
/** Which hosts of the inventory the playbook may touch (ansible's --limit). Never empty: an update run is always for this one server. */
|
||||
limit: string;
|
||||
/** How long the maintenance window lasts at most; it ends sooner when the run does. */
|
||||
windowMinutes: number;
|
||||
actor: string;
|
||||
}
|
||||
|
||||
export async function startUpdateRun(input: StartInput, now: Date = new Date()): Promise<UpdateRunRow> {
|
||||
const [server] = await db.select().from(servers).where(eq(servers.id, input.serverId)).limit(1);
|
||||
if (!server) throw new UpdateRunError(404, "server_not_found", "That server doesn't exist.");
|
||||
if (!server.lastSeenAt) throw new UpdateRunError(400, "never_reported", "This server has never reported, so there is nothing to wait for after the run.");
|
||||
|
||||
const [busy] = await db
|
||||
.select({ id: updateRuns.id })
|
||||
.from(updateRuns)
|
||||
.where(and(eq(updateRuns.serverId, server.id), inArray(updateRuns.status, ACTIVE_STATUSES)))
|
||||
.limit(1);
|
||||
if (busy) throw new UpdateRunError(409, "already_running", `An update run for ${server.name} is already under way.`);
|
||||
|
||||
const adapter = await semaphoreFor(input.integrationId);
|
||||
if (!adapter) throw new UpdateRunError(400, "no_semaphore", "That Semaphore integration doesn't exist or isn't switched on.");
|
||||
|
||||
let templateName: string;
|
||||
try {
|
||||
const { templates } = await adapter.checkTemplates();
|
||||
const template = templates.find((t) => t.projectId === input.projectId && t.id === input.templateId);
|
||||
if (!template) throw new UpdateRunError(404, "template_not_found", "That template isn't in Semaphore (any more).");
|
||||
templateName = template.name;
|
||||
} catch (err) {
|
||||
if (err instanceof UpdateRunError) throw err;
|
||||
throw new UpdateRunError(502, "semaphore_unreachable", `Couldn't reach Semaphore: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
// Quiet first, so nothing alerts about the server being worked on — and if the playbook can't be started the quiet is taken back.
|
||||
const started = await startOrExtendWindow("server", server.id, input.windowMinutes, `Update run: ${templateName}`, input.actor, now);
|
||||
let task: SemaphoreTask;
|
||||
try {
|
||||
task = await adapter.runTemplate(input.projectId, input.templateId, { limit: input.limit, message: `Update run for ${server.name}, started in Homelab Manager by ${input.actor} (limited to ${input.limit})` });
|
||||
} catch (err) {
|
||||
if (started && !started.extended) await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(eq(maintenanceWindows.id, started.window.id));
|
||||
invalidateAlertsCache();
|
||||
throw new UpdateRunError(502, "semaphore_failed", `Semaphore wouldn't start it: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
invalidateAlertsCache();
|
||||
|
||||
const updates = parsePendingUpdates(server.pendingUpdates);
|
||||
const [run] = await db
|
||||
.insert(updateRuns)
|
||||
.values({
|
||||
serverId: server.id,
|
||||
integrationId: input.integrationId,
|
||||
projectId: input.projectId,
|
||||
templateId: input.templateId,
|
||||
templateName,
|
||||
hostLimit: input.limit,
|
||||
taskId: task.id,
|
||||
taskStatus: task.status,
|
||||
windowId: started?.window.id ?? null,
|
||||
windowCreated: started ? !started.extended : false,
|
||||
status: "running",
|
||||
message: "The playbook has been started.",
|
||||
startedBy: input.actor,
|
||||
startedAt: now.toISOString(),
|
||||
updatesBefore: updates?.count ?? null,
|
||||
})
|
||||
.returning();
|
||||
await recordAudit({ category: "update_run", action: "start", targetType: "server", targetId: server.id, detail: { run: run.id, template: templateName, limit: input.limit, task: task.id, by: input.actor } });
|
||||
return run;
|
||||
}
|
||||
|
||||
// ─── Following ──────────────────────────────────────────────────────────────
|
||||
|
||||
/** Ends the maintenance window an update run made (one it only extended belongs to whoever made it, and is left to run out). */
|
||||
async function endWindow(run: UpdateRunRow, now: Date): Promise<void> {
|
||||
if (!run.windowCreated || run.windowId === null) return;
|
||||
await db
|
||||
.update(maintenanceWindows)
|
||||
.set({ endsAt: now.toISOString() })
|
||||
.where(and(eq(maintenanceWindows.id, run.windowId)));
|
||||
invalidateAlertsCache();
|
||||
}
|
||||
|
||||
async function finish(run: UpdateRunRow, status: "done" | "failed" | "cancelled", message: string, now: Date, extra: Partial<UpdateRunRow> = {}): Promise<void> {
|
||||
await endWindow(run, now);
|
||||
await db.update(updateRuns).set({ status, message, finishedAt: now.toISOString(), ...extra }).where(eq(updateRuns.id, run.id));
|
||||
const [server] = await db.select({ name: servers.name }).from(servers).where(eq(servers.id, run.serverId)).limit(1);
|
||||
await recordAudit({ category: "update_run", action: status === "done" ? "finished" : status, targetType: "server", targetId: run.serverId, detail: { run: run.id, template: run.templateName, message } });
|
||||
if (status !== "cancelled") {
|
||||
await notifyUpdateRunFinished({ serverName: server?.name ?? `#${run.serverId}`, templateName: run.templateName, ok: status === "done", message }).catch((err) => console.error("[updateRuns] couldn't send the result:", err));
|
||||
}
|
||||
}
|
||||
|
||||
const updatesWords = (n: number) => (n === 1 ? "1 update" : `${n} updates`);
|
||||
|
||||
/** What a finished run says about the server, from the report that came after it. */
|
||||
export function summarizeOutcome(before: number | null, after: number | null, reboot: boolean): string {
|
||||
const parts: string[] = [];
|
||||
if (after !== null) {
|
||||
parts.push(after === 0 ? "No updates are waiting now" : `${updatesWords(after)} still waiting`);
|
||||
if (before !== null && before > after) parts[0] += ` (${updatesWords(before - after)} installed)`;
|
||||
} else {
|
||||
parts.push("The server reported again, but doesn't say what is waiting");
|
||||
}
|
||||
parts.push(reboot ? "a restart is still needed" : "no restart is needed");
|
||||
return `${parts.join("; ")}.`;
|
||||
}
|
||||
|
||||
async function advance(run: UpdateRunRow, now: Date): Promise<void> {
|
||||
if (now.getTime() - Date.parse(run.startedAt) > RUN_LIMIT_MS) {
|
||||
await finish(run, "failed", "It took longer than three hours, so it is no longer being followed. Check Semaphore.", now);
|
||||
return;
|
||||
}
|
||||
|
||||
if (run.status === "running") {
|
||||
const adapter = await semaphoreFor(run.integrationId);
|
||||
let task: SemaphoreTask | null = null;
|
||||
try {
|
||||
task = adapter && run.taskId !== null ? await adapter.getTask(run.projectId, run.taskId) : null;
|
||||
} catch {
|
||||
task = null;
|
||||
}
|
||||
if (!task) {
|
||||
const errors = run.pollErrors + 1;
|
||||
if (errors >= MAX_POLL_ERRORS) await finish(run, "failed", "Lost contact with Semaphore, so it is no longer being followed. Check Semaphore.", now, { pollErrors: errors });
|
||||
else await db.update(updateRuns).set({ pollErrors: errors }).where(eq(updateRuns.id, run.id));
|
||||
return;
|
||||
}
|
||||
if (!FINISHED_TASK.has(task.status)) {
|
||||
if (task.status !== run.taskStatus || run.pollErrors > 0) await db.update(updateRuns).set({ taskStatus: task.status, pollErrors: 0 }).where(eq(updateRuns.id, run.id));
|
||||
return;
|
||||
}
|
||||
if (task.status === "success") {
|
||||
const ended = task.end ? new Date(task.end).toISOString() : now.toISOString();
|
||||
await db.update(updateRuns).set({ status: "waiting_for_host", taskStatus: task.status, taskEndedAt: ended, pollErrors: 0, message: "The playbook finished. Waiting for the server to report again." }).where(eq(updateRuns.id, run.id));
|
||||
return;
|
||||
}
|
||||
const why = task.status === "error" ? "The playbook failed in Semaphore" : task.status === "rejected" ? "The run was rejected in Semaphore" : "The run was stopped in Semaphore";
|
||||
await finish(run, "failed", `${why} (run #${run.taskId}). The server is no longer under maintenance.`, now, { taskStatus: task.status });
|
||||
return;
|
||||
}
|
||||
|
||||
// waiting_for_host: the agent has reported since the playbook ended (and, where it reports updates, those are newer than the run too).
|
||||
const ended = Date.parse(run.taskEndedAt ?? run.startedAt);
|
||||
const [server] = await db.select().from(servers).where(eq(servers.id, run.serverId)).limit(1);
|
||||
if (!server) {
|
||||
await finish(run, "failed", "The server was removed.", now);
|
||||
return;
|
||||
}
|
||||
const reported = server.lastSeenAt ? Date.parse(server.lastSeenAt) > ended : false;
|
||||
const updates = parsePendingUpdates(server.pendingUpdates);
|
||||
const updatesFresh = updates ? Date.parse(updates.reportedAt) > ended : true;
|
||||
if (reported && updatesFresh) {
|
||||
await finish(run, "done", summarizeOutcome(run.updatesBefore, updates?.count ?? null, updates?.rebootRequired ?? false), now, {
|
||||
hostReportedAt: server.lastSeenAt,
|
||||
updatesAfter: updates?.count ?? null,
|
||||
rebootRequired: updates?.rebootRequired ?? false,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (now.getTime() - ended > HOST_WAIT_MS) {
|
||||
await finish(run, "failed", `The playbook finished, but the server hasn't reported again in ${Math.round(HOST_WAIT_MS / 60_000)} minutes. Check that it came back.`, now);
|
||||
}
|
||||
}
|
||||
|
||||
/** One pass over every run that isn't over. Returns how many it looked at (for the tests). */
|
||||
export async function advanceUpdateRuns(now: Date = new Date()): Promise<number> {
|
||||
const active = await db.select().from(updateRuns).where(inArray(updateRuns.status, ACTIVE_STATUSES));
|
||||
for (const run of active) {
|
||||
try {
|
||||
await advance(run, now);
|
||||
} catch (err) {
|
||||
console.error(`[updateRuns] run ${run.id}:`, err);
|
||||
}
|
||||
}
|
||||
return active.length;
|
||||
}
|
||||
|
||||
export async function cancelUpdateRun(id: number, actor: string, now: Date = new Date()): Promise<UpdateRunRow | null> {
|
||||
const [run] = await db.select().from(updateRuns).where(eq(updateRuns.id, id)).limit(1);
|
||||
if (!run) return null;
|
||||
if (!ACTIVE_STATUSES.includes(run.status as UpdateRunStatus)) throw new UpdateRunError(409, "not_active", "That run is already over.");
|
||||
await finish(run, "cancelled", `Stopped being followed by ${actor}. A playbook already started keeps running in Semaphore.`, now);
|
||||
const [after] = await db.select().from(updateRuns).where(eq(updateRuns.id, id)).limit(1);
|
||||
return after;
|
||||
}
|
||||
|
||||
/** Recent runs, those still going first; for one server or for all. */
|
||||
export async function listUpdateRuns(options: { serverId?: number; limit?: number } = {}): Promise<(UpdateRunRow & { serverName: string | null })[]> {
|
||||
const rows = await db
|
||||
.select({ run: updateRuns, serverName: servers.name })
|
||||
.from(updateRuns)
|
||||
.leftJoin(servers, eq(servers.id, updateRuns.serverId))
|
||||
.where(options.serverId !== undefined ? eq(updateRuns.serverId, options.serverId) : undefined)
|
||||
.orderBy(desc(updateRuns.startedAt))
|
||||
.limit(options.limit ?? 30);
|
||||
const out = rows.map((r) => ({ ...r.run, serverName: r.serverName }));
|
||||
return [...out.filter((r) => ACTIVE_STATUSES.includes(r.status as UpdateRunStatus)), ...out.filter((r) => !ACTIVE_STATUSES.includes(r.status as UpdateRunStatus))];
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
/** Follows runs that are under way, every twenty seconds — including ones that were under way when this app last stopped. */
|
||||
export function initUpdateRuns(): void {
|
||||
if (timer) return;
|
||||
timer = setInterval(() => void advanceUpdateRuns().catch((err) => console.error("[updateRuns]", err)), POLL_INTERVAL_MS);
|
||||
}
|
||||
Reference in new issue
Block a user