Stale-backup alerts, per-server alert limits, and update runs from the Updates page

- Backups: a guest whose newest backup is older than its limit is an alert (backup kind),
  in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds)
- Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent,
  forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the
  global ones were; saving clears the alerts cache
- Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under
  maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and
  reports what is left and whether a restart is due; poller survives restarts
- Notification toggle for update-run results; tests and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 01:25:09 +02:00
1 parent 5dc6cea69c
commit 7a5b623444
42 files changed
+7561 -57

No files matched your search

+2
View File
@@ -19,6 +19,7 @@ import TimeAgo from "../components/TimeAgo";
import UptimeRibbon from "../components/UptimeRibbon";
import RunsOn from "../components/RunsOn";
import HostHealthCard from "../components/HostHealthCard";
import ServerThresholds from "../components/ServerThresholds";
import RunPlaybook from "../components/RunPlaybook";
import RunbookPanel from "../components/RunbookPanel";
import UpdatesBadge from "../components/UpdatesBadge";
@@ -810,6 +811,7 @@ export default function ServerDetail({ user }: { user: CurrentUser }) {
<HostHealthCard health={server.hostHealth} />
{hardware.source === "agent" && <ServerTrends serverId={serverId} />}
<ServerThresholds key={JSON.stringify(server.thresholds)} server={server} defaults={detail.thresholdDefaults} disks={(hardware.disks ?? []).map((d) => d.mount)} canEdit={isAdmin} onChanged={() => void loadDetail()} />
{(user.role === "admin" || user.role === "operator") && <ServerActivity serverId={serverId} />}
<div className="card mb-3">
<div className="card-header">
+33 -5
View File
@@ -1,6 +1,6 @@
import { useEffect, useState } from "react";
import { Link } from "react-router-dom";
import { api, type AgentLatest, type DockhandContainer, type UpdatesServer } from "../api/client";
import { api, type AgentLatest, type CurrentUser, type DockhandContainer, type UpdatesServer } from "../api/client";
import { agentState, agentVersionWords } from "../utils/agent";
import SortableTh from "../components/SortableTh";
import TableToolbar from "../components/TableToolbar";
@@ -13,6 +13,8 @@ import { useTableSearch } from "../hooks/useTableSearch";
import { downloadCsv } from "../utils/csv";
import { readableError } from "../utils/errors";
import ExportTableButton from "../components/ExportTableButton";
import UpdateRunDrawer from "../components/UpdateRunDrawer";
import UpdateRunsCard from "../components/UpdateRunsCard";
interface Row {
id: number;
@@ -53,13 +55,27 @@ const DAY_MS = 86_400_000;
* whether a restart is due, and how long they have been waiting) and the Docker containers that have a newer image. It only looks —
* nothing here installs anything.
*/
export default function Updates() {
export default function Updates({ user }: { user: CurrentUser }) {
const canRun = user.role === "admin" || user.role === "operator";
// The server being updated now (its drawer is open), and a counter that makes the runs card look again after one was started.
const [updating, setUpdating] = useState<UpdatesServer | null>(null);
const [runsKey, setRunsKey] = useState(0);
const [servers, setServers] = useState<UpdatesServer[] | null>(null);
const [agentLatest, setAgentLatest] = useState<AgentLatest | null>(null);
const [containers, setContainers] = useState<(DockhandContainer & { integrationName: string })[] | null>(null);
const [error, setError] = useState<string | null>(null);
const [onlyWaiting, setOnlyWaiting] = useState(false);
function reload() {
api.servers
.updates()
.then((res) => {
setServers(res.servers);
setAgentLatest(res.agentLatest);
})
.catch(() => {});
}
useEffect(() => {
let alive = true;
api.servers
@@ -125,7 +141,7 @@ export default function Updates() {
<h2 className="page-title mb-0">Updates</h2>
</div>
<p className="text-secondary">
What the agents say is waiting to be installed on each server, and the Docker containers that have a newer image. This page only looks — it installs nothing. To apply updates, run a playbook from the server's page.
What the agents say is waiting to be installed on each server, and the Docker containers that have a newer image. {canRun ? "Update… applies a server's updates through a Semaphore playbook: it silences the server, runs the playbook, waits for the server to report again and tells you what is left." : "Operators can apply a server's updates from here."}
</p>
<ErrorNotice error={error} />
@@ -143,6 +159,8 @@ export default function Updates() {
))}
</div>
<UpdateRunsCard canEdit={canRun} reloadKey={runsKey} onFinished={reload} />
<div className="card mb-3">
<TableToolbar
search={tableSearch.query}
@@ -169,12 +187,13 @@ export default function Updates() {
<SortableTh<Row> label="Waiting since" sortKeyName="since" activeKey={sortKey} direction={sortDir} onSort={requestSort} />
<SortableTh<Row> label="Reported" sortKeyName="reportedAt" activeKey={sortKey} direction={sortDir} onSort={requestSort} />
<SortableTh<Row> label="Agent" sortKeyName="agentVersion" activeKey={sortKey} direction={sortDir} onSort={requestSort} />
{canRun && <th className="w-1">Actions</th>}
</tr>
</thead>
<tbody>
{servers === null && !error && (
<tr>
<td colSpan={6}>
<td colSpan={canRun ? 7 : 6}>
<BlockSkeleton lines={3} />
</td>
</tr>
@@ -198,11 +217,18 @@ export default function Updates() {
</span>
)}
</td>
{canRun && (
<td>
<button type="button" className="btn btn-sm btn-outline-primary" disabled={r.count === null || r.count === 0 || r.server.lastSeenAt === null} title={r.count ? "Apply this server's updates through a Semaphore playbook" : "Nothing is waiting, or the agent hasn't said"} onClick={() => setUpdating(r.server)}>
Update…
</button>
</td>
)}
</tr>
))}
{servers !== null && sorted?.length === 0 && (
<tr>
<td colSpan={6} className="text-secondary text-center">
<td colSpan={canRun ? 7 : 6} className="text-secondary text-center">
{tableSearch.active || onlyWaiting ? "Nothing matches that." : "No server has reported yet."}
</td>
</tr>
@@ -262,6 +288,8 @@ export default function Updates() {
</div>
</div>
)}
{updating && <UpdateRunDrawer server={updating} onClose={() => setUpdating(null)} onStarted={() => setRunsKey((k) => k + 1)} />}
</>
);
}
@@ -54,6 +54,7 @@ const DEFAULT_NOTIFICATIONS: NotificationEvents = {
pbsVerificationCheck: true,
healthAlerts: true,
automationAlerts: true,
updateRunResults: true,
domainExpiryCheck: true,
secretCheckTime: "08:00",
timezone: "UTC",
@@ -72,7 +73,7 @@ const DIGEST_DAYS: { value: number; label: string }[] = [
{ value: 6, label: "Saturday" },
{ value: 0, label: "Sunday" },
];
const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 };
const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 };
type TestResult = { ok: boolean; message: string } | null;
type EventKey = { [K in keyof NotificationEvents]: NotificationEvents[K] extends boolean ? K : never }[keyof NotificationEvents];
@@ -688,7 +689,16 @@ export default function NotificationSettings() {
<h3 className="card-title">Backups and updates</h3>
</div>
<div className="card-body">
{toggle("proxmoxBackupCheck", "Proxmox backup failed or a guest has no coverage")}
{toggle("proxmoxBackupCheck", "Proxmox backup failed, a guest has no coverage, or its newest backup is too old")}
<NumberField
label="A guest's backup is too old after"
value={healthChecks.backupStaleHours}
min={0}
max={2160}
unit="hours"
hint="Shown on the Alerts page for every guest whose newest backup (on Proxmox or a Backup Server) is older than this, and sent once a day with the other backup messages. A server can have its own limit. 0 turns it off."
onChange={(v) => setHealthChecks((h) => ({ ...h, backupStaleHours: v }))}
/>
{toggle("pbsVerificationCheck", "Proxmox Backup Server snapshot failed verification")}
{toggle("dockerUpdateCheck", "Docker image update available")}
</div>
@@ -713,6 +723,7 @@ export default function NotificationSettings() {
onChange={(v) => setNotifications((n) => ({ ...n, integrationFailureThreshold: v }))}
/>
<div className="mt-3">{toggle("automationAlerts", "Semaphore template or Gitea workflow run failed")}</div>
<div className="mt-3">{toggle("updateRunResults", "An update run you started finished, or didn't")}</div>
</div>
</div>
</div>