Stale-backup alerts, per-server alert limits, and update runs from the Updates page

- Backups: a guest whose newest backup is older than its limit is an alert (backup kind),
  in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds)
- Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent,
  forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the
  global ones were; saving clears the alerts cache
- Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under
  maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and
  reports what is left and whether a restart is due; poller survives restarts
- Notification toggle for update-run results; tests and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 01:25:09 +02:00
1 parent 5dc6cea69c
commit 7a5b623444
42 files changed
+7561 -57

No files matched your search

+39
View File
@@ -300,6 +300,10 @@ export const servers = sqliteTable("servers", {
// What the agent says about the machine's own health, as JSON: {failedUnits, smart, zfs, reportedAt} — see services/hostHealth.ts. Null until it says.
hostHealth: text("host_health"),
// This server's own alert thresholds, as JSON: {serverOfflineMinutes, diskUsagePercent, diskForecastDays, backupStaleHours, mounts: {"/data": {percent, ignore}}} — see
// services/serverThresholds.ts. Whatever isn't there follows Settings. Null when it follows Settings entirely.
thresholds: text("thresholds"),
// Which version of the agent script is running on it, as the agent says in each report (see agent/*/report-tasks.*). Null until it says:
// an agent that predates the number being reported is version 1.
agentVersion: integer("agent_version"),
@@ -568,6 +572,41 @@ export const runbooks = sqliteTable(
(t) => [index("runbooks_target_idx").on(t.targetType, t.targetId)],
);
// ─── Update runs: apply a server's updates and see it through ──────────────
// A playbook run in Semaphore for one server, wrapped in a maintenance window and followed until the server reports again — see
// services/updateRuns.ts. status: running | waiting_for_host | done | failed | cancelled.
export const updateRuns = sqliteTable(
"update_runs",
{
id: integer("id").primaryKey({ autoIncrement: true }),
serverId: integer("server_id")
.notNull()
.references(() => servers.id, { onDelete: "cascade" }),
integrationId: integer("integration_id").notNull(), // the Semaphore integration; no FK — one removed later leaves the history readable
projectId: integer("project_id").notNull(),
templateId: integer("template_id").notNull(),
templateName: text("template_name").notNull(),
hostLimit: text("host_limit").notNull(), // ansible's --limit
taskId: integer("task_id"),
taskStatus: text("task_status"), // Semaphore's own word for the run
windowId: integer("window_id"), // the maintenance window put round it
windowCreated: integer("window_created", { mode: "boolean" }).notNull().default(false), // false if it only extended one that was already there (which is then left alone)
status: text("status").notNull(),
message: text("message"),
startedBy: text("started_by"),
startedAt: text("started_at").notNull(), // ISO
taskEndedAt: text("task_ended_at"), // ISO — when the playbook finished
hostReportedAt: text("host_reported_at"), // ISO — the report that came after it
finishedAt: text("finished_at"), // ISO
updatesBefore: integer("updates_before"),
updatesAfter: integer("updates_after"),
rebootRequired: integer("reboot_required", { mode: "boolean" }).notNull().default(false),
pollErrors: integer("poll_errors").notNull().default(0),
},
(t) => [index("update_runs_server_idx").on(t.serverId, t.startedAt), index("update_runs_status_idx").on(t.status)],
);
// ─── Inventory: the hardware behind it all ──────────────────────────────────
export const assetKinds = ["server", "storage", "network", "power", "client", "other"] as const;