Stale-backup alerts, per-server alert limits, and update runs from the Updates page

- Backups: a guest whose newest backup is older than its limit is an alert (backup kind),
  in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds)
- Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent,
  forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the
  global ones were; saving clears the alerts cache
- Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under
  maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and
  reports what is left and whether a restart is due; poller survives restarts
- Notification toggle for update-run results; tests and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 01:25:09 +02:00
1 parent 5dc6cea69c
commit 7a5b623444
42 files changed
+7561 -57

No files matched your search

+56
View File
@@ -383,6 +383,7 @@ export interface NotificationEvents {
pbsVerificationCheck: boolean;
healthAlerts: boolean;
automationAlerts: boolean;
updateRunResults: boolean;
domainExpiryCheck: boolean;
secretCheckTime: string;
timezone: string;
@@ -411,6 +412,8 @@ export interface HealthCheckSettings {
domainWarnDays: number;
/** A disk on course to be full within this many days is flagged on the Alerts page; 0 is off. */
diskForecastDays: number;
/** A Proxmox guest whose newest backup is older than this many hours is raised as stale; 0 is off. */
backupStaleHours: number;
}
export interface QuietHoursSettings {
@@ -531,6 +534,8 @@ export interface ServerRecord {
agentVersion: number | null;
/** Failed services, disk health and ZFS pools, as the agent says; null until it says anything. */
hostHealth: HostHealth | null;
/** This server's own alert limits; null when it follows Settings entirely. */
thresholds: ServerThresholds | null;
}
/** The agent version a new install would be, per system; null when this app can't tell. */
@@ -557,6 +562,46 @@ export interface PendingUpdates {
}
/** One server on the Updates page. */
/** A server's own alert limits; whatever is left out follows Settings. */
export interface ServerThresholds {
serverOfflineMinutes?: number;
diskUsagePercent?: number;
diskForecastDays?: number;
backupStaleHours?: number;
mounts?: Record<string, { percent?: number; ignore?: boolean }>;
}
export interface UpdateRun {
id: number;
serverId: number;
serverName: string | null;
integrationId: number;
template: string;
limit: string;
taskId: number | null;
/** Semaphore's own word for the run. */
taskStatus: string | null;
status: "running" | "waiting_for_host" | "done" | "failed" | "cancelled";
active: boolean;
message: string | null;
startedBy: string | null;
startedAt: string;
taskEndedAt: string | null;
finishedAt: string | null;
updatesBefore: number | null;
updatesAfter: number | null;
rebootRequired: boolean;
}
export interface UpdateRunInput {
serverId: number;
integrationId: number;
projectId: number;
templateId: number;
limit: string;
windowMinutes: number;
}
export interface UpdatesServer {
id: number;
name: string;
@@ -615,6 +660,8 @@ export interface ServerDetail {
links: ServerLink[];
relations: { parent: ServerRelation | null; children: ServerRelation[] };
agentLatest: AgentLatest;
/** The limits in Settings, which this server follows wherever it has none of its own. */
thresholdDefaults: HealthCheckSettings;
}
/** A token that opens /metrics for Prometheus, as the list shows it (never the secret). */
@@ -648,6 +695,8 @@ export type CoverageStatus = "ok" | "stale" | "failed" | "none" | "unknown";
/** One Proxmox guest and how well it is backed up. */
export interface CoverageGuest {
/** How old its newest backup may be before it counts as stale, in hours — the limit that applied to it (0 = never). */
staleAfterHours: number;
integrationId: number;
integrationName: string;
node: string;
@@ -865,6 +914,8 @@ export interface ServerUpdateInput {
parentServerId?: number | null;
/** True says it runs on nothing, which stops the guess from its Proxmox node. */
noParent?: boolean;
/** Its own alert limits; null goes back to following Settings. */
thresholds?: ServerThresholds | null;
}
/** A maintenance that repeats: "every Sunday 02:00–04:00 on these servers". */
@@ -1443,6 +1494,11 @@ export const api = {
remove: (id: number) => request<void>(`/api/event-webhooks/${id}`, { method: "DELETE" }),
test: (id: number) => request<{ ok: boolean; result: string }>(`/api/event-webhooks/${id}/test`, { method: "POST" }),
},
updateRuns: {
list: (serverId?: number) => request<{ runs: UpdateRun[] }>(`/api/update-runs${serverId ? `?serverId=${serverId}` : ""}`),
start: (data: UpdateRunInput) => request<{ run: UpdateRun }>("/api/update-runs", { method: "POST", body: JSON.stringify(data) }),
cancel: (id: number) => request<{ status: string }>(`/api/update-runs/${id}/cancel`, { method: "POST" }),
},
backups: {
coverage: (staleDays: number, refresh = false) => request<BackupCoverage>(`/api/backups/coverage?staleDays=${staleDays}${refresh ? "&refresh=1" : ""}`),
},