Stale-backup alerts, per-server alert limits, and update runs from the Updates page
- Backups: a guest whose newest backup is older than its limit is an alert (backup kind), in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds) - Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent, forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the global ones were; saving clears the alerts cache - Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and reports what is left and whether a restart is due; poller survives restarts - Notification toggle for update-run results; tests and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5dc6cea69c
commit
7a5b623444
42 files changed
+7561
-57
No files matched your search
@@ -19,6 +19,7 @@ import TimeAgo from "../components/TimeAgo";
|
||||
import UptimeRibbon from "../components/UptimeRibbon";
|
||||
import RunsOn from "../components/RunsOn";
|
||||
import HostHealthCard from "../components/HostHealthCard";
|
||||
import ServerThresholds from "../components/ServerThresholds";
|
||||
import RunPlaybook from "../components/RunPlaybook";
|
||||
import RunbookPanel from "../components/RunbookPanel";
|
||||
import UpdatesBadge from "../components/UpdatesBadge";
|
||||
@@ -810,6 +811,7 @@ export default function ServerDetail({ user }: { user: CurrentUser }) {
|
||||
|
||||
<HostHealthCard health={server.hostHealth} />
|
||||
{hardware.source === "agent" && <ServerTrends serverId={serverId} />}
|
||||
<ServerThresholds key={JSON.stringify(server.thresholds)} server={server} defaults={detail.thresholdDefaults} disks={(hardware.disks ?? []).map((d) => d.mount)} canEdit={isAdmin} onChanged={() => void loadDetail()} />
|
||||
{(user.role === "admin" || user.role === "operator") && <ServerActivity serverId={serverId} />}
|
||||
<div className="card mb-3">
|
||||
<div className="card-header">
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Link } from "react-router-dom";
|
||||
import { api, type AgentLatest, type DockhandContainer, type UpdatesServer } from "../api/client";
|
||||
import { api, type AgentLatest, type CurrentUser, type DockhandContainer, type UpdatesServer } from "../api/client";
|
||||
import { agentState, agentVersionWords } from "../utils/agent";
|
||||
import SortableTh from "../components/SortableTh";
|
||||
import TableToolbar from "../components/TableToolbar";
|
||||
@@ -13,6 +13,8 @@ import { useTableSearch } from "../hooks/useTableSearch";
|
||||
import { downloadCsv } from "../utils/csv";
|
||||
import { readableError } from "../utils/errors";
|
||||
import ExportTableButton from "../components/ExportTableButton";
|
||||
import UpdateRunDrawer from "../components/UpdateRunDrawer";
|
||||
import UpdateRunsCard from "../components/UpdateRunsCard";
|
||||
|
||||
interface Row {
|
||||
id: number;
|
||||
@@ -53,13 +55,27 @@ const DAY_MS = 86_400_000;
|
||||
* whether a restart is due, and how long they have been waiting) and the Docker containers that have a newer image. It only looks —
|
||||
* nothing here installs anything.
|
||||
*/
|
||||
export default function Updates() {
|
||||
export default function Updates({ user }: { user: CurrentUser }) {
|
||||
const canRun = user.role === "admin" || user.role === "operator";
|
||||
// The server being updated now (its drawer is open), and a counter that makes the runs card look again after one was started.
|
||||
const [updating, setUpdating] = useState<UpdatesServer | null>(null);
|
||||
const [runsKey, setRunsKey] = useState(0);
|
||||
const [servers, setServers] = useState<UpdatesServer[] | null>(null);
|
||||
const [agentLatest, setAgentLatest] = useState<AgentLatest | null>(null);
|
||||
const [containers, setContainers] = useState<(DockhandContainer & { integrationName: string })[] | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [onlyWaiting, setOnlyWaiting] = useState(false);
|
||||
|
||||
function reload() {
|
||||
api.servers
|
||||
.updates()
|
||||
.then((res) => {
|
||||
setServers(res.servers);
|
||||
setAgentLatest(res.agentLatest);
|
||||
})
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
let alive = true;
|
||||
api.servers
|
||||
@@ -125,7 +141,7 @@ export default function Updates() {
|
||||
<h2 className="page-title mb-0">Updates</h2>
|
||||
</div>
|
||||
<p className="text-secondary">
|
||||
What the agents say is waiting to be installed on each server, and the Docker containers that have a newer image. This page only looks — it installs nothing. To apply updates, run a playbook from the server's page.
|
||||
What the agents say is waiting to be installed on each server, and the Docker containers that have a newer image. {canRun ? "Update… applies a server's updates through a Semaphore playbook: it silences the server, runs the playbook, waits for the server to report again and tells you what is left." : "Operators can apply a server's updates from here."}
|
||||
</p>
|
||||
<ErrorNotice error={error} />
|
||||
|
||||
@@ -143,6 +159,8 @@ export default function Updates() {
|
||||
))}
|
||||
</div>
|
||||
|
||||
<UpdateRunsCard canEdit={canRun} reloadKey={runsKey} onFinished={reload} />
|
||||
|
||||
<div className="card mb-3">
|
||||
<TableToolbar
|
||||
search={tableSearch.query}
|
||||
@@ -169,12 +187,13 @@ export default function Updates() {
|
||||
<SortableTh<Row> label="Waiting since" sortKeyName="since" activeKey={sortKey} direction={sortDir} onSort={requestSort} />
|
||||
<SortableTh<Row> label="Reported" sortKeyName="reportedAt" activeKey={sortKey} direction={sortDir} onSort={requestSort} />
|
||||
<SortableTh<Row> label="Agent" sortKeyName="agentVersion" activeKey={sortKey} direction={sortDir} onSort={requestSort} />
|
||||
{canRun && <th className="w-1">Actions</th>}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{servers === null && !error && (
|
||||
<tr>
|
||||
<td colSpan={6}>
|
||||
<td colSpan={canRun ? 7 : 6}>
|
||||
<BlockSkeleton lines={3} />
|
||||
</td>
|
||||
</tr>
|
||||
@@ -198,11 +217,18 @@ export default function Updates() {
|
||||
</span>
|
||||
)}
|
||||
</td>
|
||||
{canRun && (
|
||||
<td>
|
||||
<button type="button" className="btn btn-sm btn-outline-primary" disabled={r.count === null || r.count === 0 || r.server.lastSeenAt === null} title={r.count ? "Apply this server's updates through a Semaphore playbook" : "Nothing is waiting, or the agent hasn't said"} onClick={() => setUpdating(r.server)}>
|
||||
Update…
|
||||
</button>
|
||||
</td>
|
||||
)}
|
||||
</tr>
|
||||
))}
|
||||
{servers !== null && sorted?.length === 0 && (
|
||||
<tr>
|
||||
<td colSpan={6} className="text-secondary text-center">
|
||||
<td colSpan={canRun ? 7 : 6} className="text-secondary text-center">
|
||||
{tableSearch.active || onlyWaiting ? "Nothing matches that." : "No server has reported yet."}
|
||||
</td>
|
||||
</tr>
|
||||
@@ -262,6 +288,8 @@ export default function Updates() {
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{updating && <UpdateRunDrawer server={updating} onClose={() => setUpdating(null)} onStarted={() => setRunsKey((k) => k + 1)} />}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -54,6 +54,7 @@ const DEFAULT_NOTIFICATIONS: NotificationEvents = {
|
||||
pbsVerificationCheck: true,
|
||||
healthAlerts: true,
|
||||
automationAlerts: true,
|
||||
updateRunResults: true,
|
||||
domainExpiryCheck: true,
|
||||
secretCheckTime: "08:00",
|
||||
timezone: "UTC",
|
||||
@@ -72,7 +73,7 @@ const DIGEST_DAYS: { value: number; label: string }[] = [
|
||||
{ value: 6, label: "Saturday" },
|
||||
{ value: 0, label: "Sunday" },
|
||||
];
|
||||
const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 };
|
||||
const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 };
|
||||
|
||||
type TestResult = { ok: boolean; message: string } | null;
|
||||
type EventKey = { [K in keyof NotificationEvents]: NotificationEvents[K] extends boolean ? K : never }[keyof NotificationEvents];
|
||||
@@ -688,7 +689,16 @@ export default function NotificationSettings() {
|
||||
<h3 className="card-title">Backups and updates</h3>
|
||||
</div>
|
||||
<div className="card-body">
|
||||
{toggle("proxmoxBackupCheck", "Proxmox backup failed or a guest has no coverage")}
|
||||
{toggle("proxmoxBackupCheck", "Proxmox backup failed, a guest has no coverage, or its newest backup is too old")}
|
||||
<NumberField
|
||||
label="A guest's backup is too old after"
|
||||
value={healthChecks.backupStaleHours}
|
||||
min={0}
|
||||
max={2160}
|
||||
unit="hours"
|
||||
hint="Shown on the Alerts page for every guest whose newest backup (on Proxmox or a Backup Server) is older than this, and sent once a day with the other backup messages. A server can have its own limit. 0 turns it off."
|
||||
onChange={(v) => setHealthChecks((h) => ({ ...h, backupStaleHours: v }))}
|
||||
/>
|
||||
{toggle("pbsVerificationCheck", "Proxmox Backup Server snapshot failed verification")}
|
||||
{toggle("dockerUpdateCheck", "Docker image update available")}
|
||||
</div>
|
||||
@@ -713,6 +723,7 @@ export default function NotificationSettings() {
|
||||
onChange={(v) => setNotifications((n) => ({ ...n, integrationFailureThreshold: v }))}
|
||||
/>
|
||||
<div className="mt-3">{toggle("automationAlerts", "Semaphore template or Gitea workflow run failed")}</div>
|
||||
<div className="mt-3">{toggle("updateRunResults", "An update run you started finished, or didn't")}</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Reference in new issue
Block a user