Stale-backup alerts, per-server alert limits, and update runs from the Updates page
- Backups: a guest whose newest backup is older than its limit is an alert (backup kind), in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds) - Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent, forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the global ones were; saving clears the alerts cache - Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and reports what is left and whether a restart is due; poller survives restarts - Notification toggle for update-run results; tests and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5dc6cea69c
commit
7a5b623444
42 files changed
+7561
-57
No files matched your search
+1
-1
@@ -141,7 +141,7 @@ function Shelled({ user }: { user: CurrentUser }) {
|
||||
<Route path="/consistency" element={<Consistency user={user} />} />
|
||||
<Route path="/domains" element={<Domains user={user} />} />
|
||||
<Route path="/maintenance" element={<Maintenance user={user} />} />
|
||||
<Route path="/updates" element={<Updates />} />
|
||||
<Route path="/updates" element={<Updates user={user} />} />
|
||||
<Route path="/inventory" element={<Inventory user={user} />} />
|
||||
<Route path="/backups" element={<Backups />} />
|
||||
<Route path="/docker" element={<Docker user={user} />} />
|
||||
|
||||
@@ -383,6 +383,7 @@ export interface NotificationEvents {
|
||||
pbsVerificationCheck: boolean;
|
||||
healthAlerts: boolean;
|
||||
automationAlerts: boolean;
|
||||
updateRunResults: boolean;
|
||||
domainExpiryCheck: boolean;
|
||||
secretCheckTime: string;
|
||||
timezone: string;
|
||||
@@ -411,6 +412,8 @@ export interface HealthCheckSettings {
|
||||
domainWarnDays: number;
|
||||
/** A disk on course to be full within this many days is flagged on the Alerts page; 0 is off. */
|
||||
diskForecastDays: number;
|
||||
/** A Proxmox guest whose newest backup is older than this many hours is raised as stale; 0 is off. */
|
||||
backupStaleHours: number;
|
||||
}
|
||||
|
||||
export interface QuietHoursSettings {
|
||||
@@ -531,6 +534,8 @@ export interface ServerRecord {
|
||||
agentVersion: number | null;
|
||||
/** Failed services, disk health and ZFS pools, as the agent says; null until it says anything. */
|
||||
hostHealth: HostHealth | null;
|
||||
/** This server's own alert limits; null when it follows Settings entirely. */
|
||||
thresholds: ServerThresholds | null;
|
||||
}
|
||||
|
||||
/** The agent version a new install would be, per system; null when this app can't tell. */
|
||||
@@ -557,6 +562,46 @@ export interface PendingUpdates {
|
||||
}
|
||||
|
||||
/** One server on the Updates page. */
|
||||
/** A server's own alert limits; whatever is left out follows Settings. */
|
||||
export interface ServerThresholds {
|
||||
serverOfflineMinutes?: number;
|
||||
diskUsagePercent?: number;
|
||||
diskForecastDays?: number;
|
||||
backupStaleHours?: number;
|
||||
mounts?: Record<string, { percent?: number; ignore?: boolean }>;
|
||||
}
|
||||
|
||||
export interface UpdateRun {
|
||||
id: number;
|
||||
serverId: number;
|
||||
serverName: string | null;
|
||||
integrationId: number;
|
||||
template: string;
|
||||
limit: string;
|
||||
taskId: number | null;
|
||||
/** Semaphore's own word for the run. */
|
||||
taskStatus: string | null;
|
||||
status: "running" | "waiting_for_host" | "done" | "failed" | "cancelled";
|
||||
active: boolean;
|
||||
message: string | null;
|
||||
startedBy: string | null;
|
||||
startedAt: string;
|
||||
taskEndedAt: string | null;
|
||||
finishedAt: string | null;
|
||||
updatesBefore: number | null;
|
||||
updatesAfter: number | null;
|
||||
rebootRequired: boolean;
|
||||
}
|
||||
|
||||
export interface UpdateRunInput {
|
||||
serverId: number;
|
||||
integrationId: number;
|
||||
projectId: number;
|
||||
templateId: number;
|
||||
limit: string;
|
||||
windowMinutes: number;
|
||||
}
|
||||
|
||||
export interface UpdatesServer {
|
||||
id: number;
|
||||
name: string;
|
||||
@@ -615,6 +660,8 @@ export interface ServerDetail {
|
||||
links: ServerLink[];
|
||||
relations: { parent: ServerRelation | null; children: ServerRelation[] };
|
||||
agentLatest: AgentLatest;
|
||||
/** The limits in Settings, which this server follows wherever it has none of its own. */
|
||||
thresholdDefaults: HealthCheckSettings;
|
||||
}
|
||||
|
||||
/** A token that opens /metrics for Prometheus, as the list shows it (never the secret). */
|
||||
@@ -648,6 +695,8 @@ export type CoverageStatus = "ok" | "stale" | "failed" | "none" | "unknown";
|
||||
|
||||
/** One Proxmox guest and how well it is backed up. */
|
||||
export interface CoverageGuest {
|
||||
/** How old its newest backup may be before it counts as stale, in hours — the limit that applied to it (0 = never). */
|
||||
staleAfterHours: number;
|
||||
integrationId: number;
|
||||
integrationName: string;
|
||||
node: string;
|
||||
@@ -865,6 +914,8 @@ export interface ServerUpdateInput {
|
||||
parentServerId?: number | null;
|
||||
/** True says it runs on nothing, which stops the guess from its Proxmox node. */
|
||||
noParent?: boolean;
|
||||
/** Its own alert limits; null goes back to following Settings. */
|
||||
thresholds?: ServerThresholds | null;
|
||||
}
|
||||
|
||||
/** A maintenance that repeats: "every Sunday 02:00–04:00 on these servers". */
|
||||
@@ -1443,6 +1494,11 @@ export const api = {
|
||||
remove: (id: number) => request<void>(`/api/event-webhooks/${id}`, { method: "DELETE" }),
|
||||
test: (id: number) => request<{ ok: boolean; result: string }>(`/api/event-webhooks/${id}/test`, { method: "POST" }),
|
||||
},
|
||||
updateRuns: {
|
||||
list: (serverId?: number) => request<{ runs: UpdateRun[] }>(`/api/update-runs${serverId ? `?serverId=${serverId}` : ""}`),
|
||||
start: (data: UpdateRunInput) => request<{ run: UpdateRun }>("/api/update-runs", { method: "POST", body: JSON.stringify(data) }),
|
||||
cancel: (id: number) => request<{ status: string }>(`/api/update-runs/${id}/cancel`, { method: "POST" }),
|
||||
},
|
||||
backups: {
|
||||
coverage: (staleDays: number, refresh = false) => request<BackupCoverage>(`/api/backups/coverage?staleDays=${staleDays}${refresh ? "&refresh=1" : ""}`),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
import { useState } from "react";
|
||||
import { api, type HealthCheckSettings, type ServerRecord, type ServerThresholds as Thresholds } from "../api/client";
|
||||
import { readableError } from "../utils/errors";
|
||||
import { toast } from "../utils/toast";
|
||||
import { ErrorNotice } from "./Notice";
|
||||
|
||||
type Field = "serverOfflineMinutes" | "diskUsagePercent" | "diskForecastDays" | "backupStaleHours";
|
||||
|
||||
const FIELDS: { key: Field; label: string; unit: string; min: number; max: number; hint: string; onlyForGuests?: boolean }[] = [
|
||||
{ key: "serverOfflineMinutes", label: "Offline after", unit: "min", min: 15, max: 10080, hint: "How long without a report before this server counts as offline — longer for a laptop that is off at night." },
|
||||
{ key: "diskUsagePercent", label: "Disk usage alert at", unit: "%", min: 50, max: 99, hint: "For every disk on this server that has no limit of its own below." },
|
||||
{ key: "diskForecastDays", label: "Warn when a disk will fill within", unit: "days", min: 0, max: 90, hint: "0 turns the forecast off for this server." },
|
||||
{ key: "backupStaleHours", label: "Its backup is too old after", unit: "hours", min: 0, max: 2160, hint: "For the Proxmox guest this server is. 0 means it is never called stale.", onlyForGuests: true },
|
||||
];
|
||||
|
||||
const has = (t: Thresholds | null) => !!t && (Object.keys(t).some((k) => k !== "mounts") || Object.keys(t.mounts ?? {}).length > 0);
|
||||
|
||||
/**
|
||||
* This server's own alert limits. Anything left empty follows Settings → Notifications, so changing the global limit still reaches every
|
||||
* server that hasn't said otherwise. A disk can have a limit of its own, or be left alone altogether. Everyone can see what is set; only
|
||||
* an administrator changes it.
|
||||
*/
|
||||
export default function ServerThresholds({ server, defaults, disks, canEdit, onChanged }: { server: ServerRecord; defaults: HealthCheckSettings; disks: string[]; canEdit: boolean; onChanged: () => void }) {
|
||||
const own = server.thresholds;
|
||||
const isGuest = server.proxmoxVmid !== null;
|
||||
|
||||
const [draft, setDraft] = useState<Record<Field, string>>(() => ({
|
||||
serverOfflineMinutes: String(own?.serverOfflineMinutes ?? ""),
|
||||
diskUsagePercent: String(own?.diskUsagePercent ?? ""),
|
||||
diskForecastDays: String(own?.diskForecastDays ?? ""),
|
||||
backupStaleHours: String(own?.backupStaleHours ?? ""),
|
||||
}));
|
||||
const [mounts, setMounts] = useState<Record<string, { percent: string; ignore: boolean }>>(() =>
|
||||
Object.fromEntries(Object.entries(own?.mounts ?? {}).map(([m, r]) => [m, { percent: String(r.percent ?? ""), ignore: !!r.ignore }])),
|
||||
);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
// The disks it reports now, plus any that have a rule but aren't reported any more (so the rule can still be taken away).
|
||||
const mountNames = [...new Set([...disks, ...Object.keys(mounts)])];
|
||||
|
||||
function build(): Thresholds | null {
|
||||
const out: Thresholds = {};
|
||||
for (const f of FIELDS) {
|
||||
const v = draft[f.key].trim();
|
||||
if (v !== "") out[f.key] = Number(v);
|
||||
}
|
||||
const rules: NonNullable<Thresholds["mounts"]> = {};
|
||||
for (const [m, r] of Object.entries(mounts)) {
|
||||
const rule: { percent?: number; ignore?: boolean } = {};
|
||||
if (r.percent.trim() !== "") rule.percent = Number(r.percent);
|
||||
if (r.ignore) rule.ignore = true;
|
||||
if (rule.percent !== undefined || rule.ignore) rules[m] = rule;
|
||||
}
|
||||
if (Object.keys(rules).length > 0) out.mounts = rules;
|
||||
return Object.keys(out).length > 0 ? out : null;
|
||||
}
|
||||
|
||||
async function save(value: Thresholds | null) {
|
||||
setBusy(true);
|
||||
setError(null);
|
||||
try {
|
||||
await api.servers.update(server.id, { thresholds: value });
|
||||
toast.success(value ? "Saved. This server now follows its own limits where it has them." : "This server follows Settings again.");
|
||||
onChanged();
|
||||
} catch (err) {
|
||||
setError(readableError(err));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (!canEdit) {
|
||||
if (!has(own)) return null;
|
||||
return (
|
||||
<div className="card mb-3">
|
||||
<div className="card-header">
|
||||
<h3 className="card-title">Alert limits</h3>
|
||||
</div>
|
||||
<div className="card-body">
|
||||
<div className="text-secondary small mb-2">This server has limits of its own; everything else follows Settings.</div>
|
||||
<ul className="mb-0">
|
||||
{FIELDS.filter((f) => own?.[f.key] !== undefined).map((f) => (
|
||||
<li key={f.key}>
|
||||
{f.label}: <strong>{own![f.key]}</strong> {f.unit}
|
||||
</li>
|
||||
))}
|
||||
{Object.entries(own?.mounts ?? {}).map(([m, r]) => (
|
||||
<li key={m}>
|
||||
<code>{m}</code>: {r.ignore ? "left alone" : `alerts at ${r.percent}%`}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="card mb-3">
|
||||
<div className="card-header">
|
||||
<h3 className="card-title">Alert limits</h3>
|
||||
{has(own) && (
|
||||
<div className="card-actions">
|
||||
<span className="badge bg-blue-lt text-blue">its own limits</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<form
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
void save(build());
|
||||
}}
|
||||
>
|
||||
<div className="card-body">
|
||||
<p className="text-secondary small">Leave a box empty to follow Settings → Notifications (shown inside the box). Only what you fill in is different for this server.</p>
|
||||
<div className="row g-3">
|
||||
{FIELDS.filter((f) => !f.onlyForGuests || isGuest || own?.[f.key] !== undefined).map((f) => (
|
||||
<div className="col-md-6" key={f.key}>
|
||||
<label className="form-label" htmlFor={`th-${f.key}`}>
|
||||
{f.label} <span className="text-secondary">({f.unit})</span>
|
||||
</label>
|
||||
<input
|
||||
id={`th-${f.key}`}
|
||||
type="number"
|
||||
className="form-control"
|
||||
min={f.min}
|
||||
max={f.max}
|
||||
step={1}
|
||||
placeholder={`Settings: ${defaults[f.key]}`}
|
||||
value={draft[f.key]}
|
||||
onChange={(e) => setDraft((d) => ({ ...d, [f.key]: e.target.value }))}
|
||||
/>
|
||||
<div className="form-hint text-secondary small mt-1">{f.hint}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{mountNames.length > 0 && (
|
||||
<div className="mt-3">
|
||||
<div className="form-label">Disks</div>
|
||||
<div className="table-responsive">
|
||||
<table className="table table-sm table-vcenter mb-0">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Disk</th>
|
||||
<th style={{ width: 180 }}>Alert at (%)</th>
|
||||
<th>Leave it alone</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{mountNames.map((m) => {
|
||||
const r = mounts[m] ?? { percent: "", ignore: false };
|
||||
const set = (patch: Partial<typeof r>) => setMounts((all) => ({ ...all, [m]: { ...r, ...patch } }));
|
||||
return (
|
||||
<tr key={m}>
|
||||
<td>
|
||||
<code>{m}</code>
|
||||
{!disks.includes(m) && <span className="text-secondary small ms-2">(not reported now)</span>}
|
||||
</td>
|
||||
<td>
|
||||
<input
|
||||
type="number"
|
||||
className="form-control form-control-sm"
|
||||
aria-label={`Alert at for ${m}`}
|
||||
min={50}
|
||||
max={99}
|
||||
step={1}
|
||||
placeholder={`${draft.diskUsagePercent.trim() || defaults.diskUsagePercent}`}
|
||||
disabled={r.ignore}
|
||||
value={r.percent}
|
||||
onChange={(e) => set({ percent: e.target.value })}
|
||||
/>
|
||||
</td>
|
||||
<td>
|
||||
<label className="form-check mb-0">
|
||||
<input type="checkbox" className="form-check-input" checked={r.ignore} onChange={(e) => set({ ignore: e.target.checked })} />
|
||||
<span className="form-check-label text-secondary">no alerts for this disk</span>
|
||||
</label>
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<div className="mt-3">
|
||||
<ErrorNotice error={error} />
|
||||
</div>
|
||||
</div>
|
||||
<div className="card-footer d-flex gap-2">
|
||||
<button type="submit" className="btn btn-primary" disabled={busy}>
|
||||
{busy ? "Saving…" : "Save"}
|
||||
</button>
|
||||
{has(own) && (
|
||||
<button type="button" className="btn" disabled={busy} onClick={() => void save(null)}>
|
||||
Follow Settings again
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Link } from "react-router-dom";
|
||||
import { api, type IntegrationSummary, type SemaphoreTemplate, type UpdatesServer } from "../api/client";
|
||||
import { confirmDialog } from "../utils/dialogs";
|
||||
import { readableError } from "../utils/errors";
|
||||
import { toast } from "../utils/toast";
|
||||
import DetailDrawer from "./DetailDrawer";
|
||||
import { BlockSkeleton } from "./Skeleton";
|
||||
import UpdatesBadge from "./UpdatesBadge";
|
||||
|
||||
interface Loaded {
|
||||
integration: IntegrationSummary;
|
||||
templates: SemaphoreTemplate[];
|
||||
}
|
||||
|
||||
const WINDOWS = [
|
||||
{ minutes: 45, label: "45 minutes" },
|
||||
{ minutes: 90, label: "1½ hours" },
|
||||
{ minutes: 180, label: "3 hours" },
|
||||
{ minutes: 360, label: "6 hours" },
|
||||
];
|
||||
|
||||
const LAST_KEY = "hlm.updateRun.template";
|
||||
const remembered = () => {
|
||||
try {
|
||||
return localStorage.getItem(LAST_KEY) ?? "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* "Update this server now": pick the playbook that applies updates, and Homelab Manager puts the server under maintenance, runs it for this
|
||||
* server only, follows it, waits for the server to report again and says what is left and whether a restart is due. It asks before it
|
||||
* starts, and says what it will do.
|
||||
*/
|
||||
export default function UpdateRunDrawer({ server, onClose, onStarted }: { server: UpdatesServer; onClose: () => void; onStarted: () => void }) {
|
||||
const [loaded, setLoaded] = useState<Loaded[] | null>(null);
|
||||
const [failed, setFailed] = useState(false);
|
||||
const [choice, setChoice] = useState(remembered);
|
||||
const [limit, setLimit] = useState(server.name);
|
||||
const [minutes, setMinutes] = useState(90);
|
||||
const [busy, setBusy] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
let alive = true;
|
||||
api.integrations
|
||||
.list()
|
||||
.then(async (res) => {
|
||||
const sems = res.integrations.filter((i) => i.type === "semaphore" && i.enabled);
|
||||
const all = await Promise.all(sems.map((integration) => api.integrations.semaphore.templates(integration.id).then((r): Loaded => ({ integration, templates: r.templates })).catch((): Loaded => ({ integration, templates: [] }))));
|
||||
if (alive) setLoaded(all);
|
||||
})
|
||||
.catch(() => alive && setFailed(true));
|
||||
return () => {
|
||||
alive = false;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const options = (loaded ?? []).flatMap((l) =>
|
||||
l.templates.map((t) => ({ key: `${l.integration.id}:${t.projectId}:${t.id}`, integration: l.integration, template: t, label: `${t.name} — ${t.projectName}${(loaded ?? []).length > 1 ? ` (${l.integration.name})` : ""}` })),
|
||||
);
|
||||
const picked = options.find((o) => o.key === choice);
|
||||
const count = server.updates?.count ?? null;
|
||||
|
||||
async function start() {
|
||||
if (!picked) return;
|
||||
const ok = await confirmDialog({
|
||||
title: `Update ${server.name} now?`,
|
||||
message: `This silences alerts for ${server.name} for up to ${WINDOWS.find((w) => w.minutes === minutes)?.label ?? `${minutes} minutes`}, runs "${picked.template.name}" in ${picked.integration.name} limited to ${limit.trim()}, and waits for the server to report again. The playbook may restart it.`,
|
||||
confirmLabel: "Start the update",
|
||||
});
|
||||
if (!ok) return;
|
||||
setBusy(true);
|
||||
try {
|
||||
await api.updateRuns.start({ serverId: server.id, integrationId: picked.integration.id, projectId: picked.template.projectId, templateId: picked.template.id, limit: limit.trim(), windowMinutes: minutes });
|
||||
try {
|
||||
localStorage.setItem(LAST_KEY, picked.key);
|
||||
} catch {
|
||||
// a remembered choice is only a convenience
|
||||
}
|
||||
toast.success(`Started updating ${server.name}.`);
|
||||
onStarted();
|
||||
onClose();
|
||||
} catch (err) {
|
||||
toast.error(readableError(err));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<DetailDrawer title={`Update ${server.name}`} subtitle={server.osType === "windows" ? "Windows" : "Linux"} onClose={onClose}>
|
||||
<div className="mb-3">
|
||||
<UpdatesBadge updates={server.updates} />
|
||||
{count !== null && count > 0 && server.updates && server.updates.security !== null && <span className="text-secondary small ms-2">{server.updates.security} of them security</span>}
|
||||
</div>
|
||||
|
||||
{failed ? (
|
||||
<div className="text-secondary">Couldn't load the integrations.</div>
|
||||
) : loaded === null ? (
|
||||
<BlockSkeleton lines={3} />
|
||||
) : loaded.length === 0 ? (
|
||||
<div className="text-secondary">
|
||||
There is no Semaphore connected. Add one under <Link to="/integrations">Integrations</Link>, with a playbook that applies updates, and this server can be updated from here.
|
||||
</div>
|
||||
) : options.length === 0 ? (
|
||||
<div className="text-secondary">No templates found in Semaphore.</div>
|
||||
) : (
|
||||
<>
|
||||
<ol className="text-secondary small ps-3 mb-3">
|
||||
<li>Alerts for this server are silenced (a maintenance window).</li>
|
||||
<li>The playbook runs in Semaphore, for this server only.</li>
|
||||
<li>Homelab Manager waits for the server to report again — it may be restarting.</li>
|
||||
<li>It tells you what is still waiting and whether a restart is due, and the window ends.</li>
|
||||
</ol>
|
||||
<div className="mb-3">
|
||||
<label className="form-label" htmlFor="ur-template">
|
||||
Playbook that applies updates
|
||||
</label>
|
||||
<select id="ur-template" className="form-select" value={choice} onChange={(e) => setChoice(e.target.value)}>
|
||||
<option value="">Choose a template…</option>
|
||||
{options.map((o) => (
|
||||
<option key={o.key} value={o.key}>
|
||||
{o.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
<div className="mb-3">
|
||||
<label className="form-label" htmlFor="ur-limit">
|
||||
Run it on — the host name as it is in the Semaphore inventory
|
||||
</label>
|
||||
<input id="ur-limit" className="form-control" value={limit} maxLength={200} onChange={(e) => setLimit(e.target.value)} placeholder="e.g. web01" />
|
||||
<div className="form-hint text-secondary small mt-1">An update run is always for this one server, never the whole inventory.</div>
|
||||
</div>
|
||||
<div className="mb-3">
|
||||
<label className="form-label" htmlFor="ur-window">
|
||||
Silence its alerts for up to
|
||||
</label>
|
||||
<select id="ur-window" className="form-select" value={minutes} onChange={(e) => setMinutes(Number(e.target.value))}>
|
||||
{WINDOWS.map((w) => (
|
||||
<option key={w.minutes} value={w.minutes}>
|
||||
{w.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<div className="form-hint text-secondary small mt-1">The window ends sooner, as soon as the run does.</div>
|
||||
</div>
|
||||
<button type="button" className="btn btn-primary" disabled={!picked || busy || limit.trim() === ""} onClick={() => void start()}>
|
||||
{busy ? "Starting…" : "Update now"}
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
</DetailDrawer>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Link } from "react-router-dom";
|
||||
import { api, type UpdateRun } from "../api/client";
|
||||
import { confirmDialog } from "../utils/dialogs";
|
||||
import { readableError } from "../utils/errors";
|
||||
import { toast } from "../utils/toast";
|
||||
import TimeAgo from "./TimeAgo";
|
||||
|
||||
const TASK_WORDS: Record<string, string> = {
|
||||
waiting: "waiting to start in Semaphore",
|
||||
starting: "starting in Semaphore",
|
||||
running: "playbook running",
|
||||
waiting_confirmation: "waiting for approval in Semaphore",
|
||||
confirmed: "approved in Semaphore",
|
||||
stopping: "stopping",
|
||||
};
|
||||
|
||||
/** What a run is doing now, in words: the badge and the sentence beside it. */
|
||||
export function describeRun(run: UpdateRun): { label: string; badge: string } {
|
||||
switch (run.status) {
|
||||
case "running":
|
||||
return { label: TASK_WORDS[run.taskStatus ?? ""] ?? "playbook running", badge: "bg-blue-lt text-blue" };
|
||||
case "waiting_for_host":
|
||||
return { label: "waiting for the server to report again", badge: "bg-blue-lt text-blue" };
|
||||
case "done":
|
||||
return { label: "finished", badge: run.rebootRequired ? "bg-yellow-lt text-yellow" : "bg-green-lt text-green" };
|
||||
case "failed":
|
||||
return { label: "didn't finish", badge: "bg-red-lt text-red" };
|
||||
default:
|
||||
return { label: "stopped being followed", badge: "bg-secondary-lt text-secondary" };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update runs under way and recently finished: for each, where it has got to (quiet → playbook → server back → done), what is still waiting
|
||||
* afterwards and whether a restart is due. Looks again every few seconds while anything is under way. Hidden when there has been none.
|
||||
* `reloadKey` makes it look straight away (after a run was started elsewhere on the page).
|
||||
*/
|
||||
export default function UpdateRunsCard({ canEdit, reloadKey, onFinished }: { canEdit: boolean; reloadKey: number; onFinished?: () => void }) {
|
||||
const [runs, setRuns] = useState<UpdateRun[] | null>(null);
|
||||
const active = (runs ?? []).some((r) => r.active);
|
||||
|
||||
useEffect(() => {
|
||||
let alive = true;
|
||||
const load = () =>
|
||||
api.updateRuns
|
||||
.list()
|
||||
.then((res) => {
|
||||
if (!alive) return;
|
||||
setRuns((before) => {
|
||||
// A run that was under way and now isn't: the page's figures have probably changed.
|
||||
if (before?.some((b) => b.active && !res.runs.find((r) => r.id === b.id)?.active)) onFinished?.();
|
||||
return res.runs;
|
||||
});
|
||||
})
|
||||
.catch(() => {});
|
||||
void load();
|
||||
if (!active) return () => void (alive = false);
|
||||
const timer = setInterval(load, 5000);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearInterval(timer);
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [reloadKey, active]);
|
||||
|
||||
async function stop(run: UpdateRun) {
|
||||
if (
|
||||
!(await confirmDialog({
|
||||
title: "Stop following this run?",
|
||||
message: `The maintenance window on ${run.serverName} ends now. A playbook already started keeps running in Semaphore — it can't be stopped from here.`,
|
||||
confirmLabel: "Stop following",
|
||||
danger: true,
|
||||
}))
|
||||
)
|
||||
return;
|
||||
try {
|
||||
await api.updateRuns.cancel(run.id);
|
||||
toast.success("No longer following it.");
|
||||
setRuns((await api.updateRuns.list()).runs);
|
||||
} catch (err) {
|
||||
toast.error(readableError(err));
|
||||
}
|
||||
}
|
||||
|
||||
if (!runs || runs.length === 0) return null;
|
||||
return (
|
||||
<div className="card mb-3">
|
||||
<div className="card-header">
|
||||
<h3 className="card-title">Update runs</h3>
|
||||
<div className="card-actions text-secondary small">{active ? "checking every few seconds" : "most recent first"}</div>
|
||||
</div>
|
||||
<div className="list-group list-group-flush">
|
||||
{runs.slice(0, 8).map((r) => {
|
||||
const w = describeRun(r);
|
||||
return (
|
||||
<div className="list-group-item" key={r.id}>
|
||||
<div className="d-flex flex-wrap align-items-center gap-2">
|
||||
<Link to={`/servers/${r.serverId}`}>
|
||||
<strong>{r.serverName ?? `server #${r.serverId}`}</strong>
|
||||
</Link>
|
||||
<span className="text-secondary">{r.template}</span>
|
||||
<span className={`badge ${w.badge}`}>{w.label}</span>
|
||||
{r.status === "done" && r.rebootRequired && <span className="badge bg-yellow-lt text-yellow">restart needed</span>}
|
||||
<span className="text-secondary small ms-auto">
|
||||
started <TimeAgo value={r.startedAt} /> by {r.startedBy ?? "someone"}
|
||||
</span>
|
||||
{canEdit && r.active && (
|
||||
<button type="button" className="btn btn-sm btn-outline-secondary" onClick={() => void stop(r)}>
|
||||
Stop following
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
{r.message && <div className="text-secondary small mt-1">{r.message}</div>}
|
||||
{r.active && (
|
||||
<div className="progress progress-sm mt-2" aria-hidden="true">
|
||||
<div className="progress-bar progress-bar-indeterminate" />
|
||||
</div>
|
||||
)}
|
||||
{r.taskId !== null && <div className="text-secondary small">Run #{r.taskId} in Semaphore — the playbook's output is there.</div>}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import TimeAgo from "../components/TimeAgo";
|
||||
import UptimeRibbon from "../components/UptimeRibbon";
|
||||
import RunsOn from "../components/RunsOn";
|
||||
import HostHealthCard from "../components/HostHealthCard";
|
||||
import ServerThresholds from "../components/ServerThresholds";
|
||||
import RunPlaybook from "../components/RunPlaybook";
|
||||
import RunbookPanel from "../components/RunbookPanel";
|
||||
import UpdatesBadge from "../components/UpdatesBadge";
|
||||
@@ -810,6 +811,7 @@ export default function ServerDetail({ user }: { user: CurrentUser }) {
|
||||
|
||||
<HostHealthCard health={server.hostHealth} />
|
||||
{hardware.source === "agent" && <ServerTrends serverId={serverId} />}
|
||||
<ServerThresholds key={JSON.stringify(server.thresholds)} server={server} defaults={detail.thresholdDefaults} disks={(hardware.disks ?? []).map((d) => d.mount)} canEdit={isAdmin} onChanged={() => void loadDetail()} />
|
||||
{(user.role === "admin" || user.role === "operator") && <ServerActivity serverId={serverId} />}
|
||||
<div className="card mb-3">
|
||||
<div className="card-header">
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Link } from "react-router-dom";
|
||||
import { api, type AgentLatest, type DockhandContainer, type UpdatesServer } from "../api/client";
|
||||
import { api, type AgentLatest, type CurrentUser, type DockhandContainer, type UpdatesServer } from "../api/client";
|
||||
import { agentState, agentVersionWords } from "../utils/agent";
|
||||
import SortableTh from "../components/SortableTh";
|
||||
import TableToolbar from "../components/TableToolbar";
|
||||
@@ -13,6 +13,8 @@ import { useTableSearch } from "../hooks/useTableSearch";
|
||||
import { downloadCsv } from "../utils/csv";
|
||||
import { readableError } from "../utils/errors";
|
||||
import ExportTableButton from "../components/ExportTableButton";
|
||||
import UpdateRunDrawer from "../components/UpdateRunDrawer";
|
||||
import UpdateRunsCard from "../components/UpdateRunsCard";
|
||||
|
||||
interface Row {
|
||||
id: number;
|
||||
@@ -53,13 +55,27 @@ const DAY_MS = 86_400_000;
|
||||
* whether a restart is due, and how long they have been waiting) and the Docker containers that have a newer image. It only looks —
|
||||
* nothing here installs anything.
|
||||
*/
|
||||
export default function Updates() {
|
||||
export default function Updates({ user }: { user: CurrentUser }) {
|
||||
const canRun = user.role === "admin" || user.role === "operator";
|
||||
// The server being updated now (its drawer is open), and a counter that makes the runs card look again after one was started.
|
||||
const [updating, setUpdating] = useState<UpdatesServer | null>(null);
|
||||
const [runsKey, setRunsKey] = useState(0);
|
||||
const [servers, setServers] = useState<UpdatesServer[] | null>(null);
|
||||
const [agentLatest, setAgentLatest] = useState<AgentLatest | null>(null);
|
||||
const [containers, setContainers] = useState<(DockhandContainer & { integrationName: string })[] | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [onlyWaiting, setOnlyWaiting] = useState(false);
|
||||
|
||||
function reload() {
|
||||
api.servers
|
||||
.updates()
|
||||
.then((res) => {
|
||||
setServers(res.servers);
|
||||
setAgentLatest(res.agentLatest);
|
||||
})
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
let alive = true;
|
||||
api.servers
|
||||
@@ -125,7 +141,7 @@ export default function Updates() {
|
||||
<h2 className="page-title mb-0">Updates</h2>
|
||||
</div>
|
||||
<p className="text-secondary">
|
||||
What the agents say is waiting to be installed on each server, and the Docker containers that have a newer image. This page only looks — it installs nothing. To apply updates, run a playbook from the server's page.
|
||||
What the agents say is waiting to be installed on each server, and the Docker containers that have a newer image. {canRun ? "Update… applies a server's updates through a Semaphore playbook: it silences the server, runs the playbook, waits for the server to report again and tells you what is left." : "Operators can apply a server's updates from here."}
|
||||
</p>
|
||||
<ErrorNotice error={error} />
|
||||
|
||||
@@ -143,6 +159,8 @@ export default function Updates() {
|
||||
))}
|
||||
</div>
|
||||
|
||||
<UpdateRunsCard canEdit={canRun} reloadKey={runsKey} onFinished={reload} />
|
||||
|
||||
<div className="card mb-3">
|
||||
<TableToolbar
|
||||
search={tableSearch.query}
|
||||
@@ -169,12 +187,13 @@ export default function Updates() {
|
||||
<SortableTh<Row> label="Waiting since" sortKeyName="since" activeKey={sortKey} direction={sortDir} onSort={requestSort} />
|
||||
<SortableTh<Row> label="Reported" sortKeyName="reportedAt" activeKey={sortKey} direction={sortDir} onSort={requestSort} />
|
||||
<SortableTh<Row> label="Agent" sortKeyName="agentVersion" activeKey={sortKey} direction={sortDir} onSort={requestSort} />
|
||||
{canRun && <th className="w-1">Actions</th>}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{servers === null && !error && (
|
||||
<tr>
|
||||
<td colSpan={6}>
|
||||
<td colSpan={canRun ? 7 : 6}>
|
||||
<BlockSkeleton lines={3} />
|
||||
</td>
|
||||
</tr>
|
||||
@@ -198,11 +217,18 @@ export default function Updates() {
|
||||
</span>
|
||||
)}
|
||||
</td>
|
||||
{canRun && (
|
||||
<td>
|
||||
<button type="button" className="btn btn-sm btn-outline-primary" disabled={r.count === null || r.count === 0 || r.server.lastSeenAt === null} title={r.count ? "Apply this server's updates through a Semaphore playbook" : "Nothing is waiting, or the agent hasn't said"} onClick={() => setUpdating(r.server)}>
|
||||
Update…
|
||||
</button>
|
||||
</td>
|
||||
)}
|
||||
</tr>
|
||||
))}
|
||||
{servers !== null && sorted?.length === 0 && (
|
||||
<tr>
|
||||
<td colSpan={6} className="text-secondary text-center">
|
||||
<td colSpan={canRun ? 7 : 6} className="text-secondary text-center">
|
||||
{tableSearch.active || onlyWaiting ? "Nothing matches that." : "No server has reported yet."}
|
||||
</td>
|
||||
</tr>
|
||||
@@ -262,6 +288,8 @@ export default function Updates() {
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{updating && <UpdateRunDrawer server={updating} onClose={() => setUpdating(null)} onStarted={() => setRunsKey((k) => k + 1)} />}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -54,6 +54,7 @@ const DEFAULT_NOTIFICATIONS: NotificationEvents = {
|
||||
pbsVerificationCheck: true,
|
||||
healthAlerts: true,
|
||||
automationAlerts: true,
|
||||
updateRunResults: true,
|
||||
domainExpiryCheck: true,
|
||||
secretCheckTime: "08:00",
|
||||
timezone: "UTC",
|
||||
@@ -72,7 +73,7 @@ const DIGEST_DAYS: { value: number; label: string }[] = [
|
||||
{ value: 6, label: "Saturday" },
|
||||
{ value: 0, label: "Sunday" },
|
||||
];
|
||||
const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 };
|
||||
const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 };
|
||||
|
||||
type TestResult = { ok: boolean; message: string } | null;
|
||||
type EventKey = { [K in keyof NotificationEvents]: NotificationEvents[K] extends boolean ? K : never }[keyof NotificationEvents];
|
||||
@@ -688,7 +689,16 @@ export default function NotificationSettings() {
|
||||
<h3 className="card-title">Backups and updates</h3>
|
||||
</div>
|
||||
<div className="card-body">
|
||||
{toggle("proxmoxBackupCheck", "Proxmox backup failed or a guest has no coverage")}
|
||||
{toggle("proxmoxBackupCheck", "Proxmox backup failed, a guest has no coverage, or its newest backup is too old")}
|
||||
<NumberField
|
||||
label="A guest's backup is too old after"
|
||||
value={healthChecks.backupStaleHours}
|
||||
min={0}
|
||||
max={2160}
|
||||
unit="hours"
|
||||
hint="Shown on the Alerts page for every guest whose newest backup (on Proxmox or a Backup Server) is older than this, and sent once a day with the other backup messages. A server can have its own limit. 0 turns it off."
|
||||
onChange={(v) => setHealthChecks((h) => ({ ...h, backupStaleHours: v }))}
|
||||
/>
|
||||
{toggle("pbsVerificationCheck", "Proxmox Backup Server snapshot failed verification")}
|
||||
{toggle("dockerUpdateCheck", "Docker image update available")}
|
||||
</div>
|
||||
@@ -713,6 +723,7 @@ export default function NotificationSettings() {
|
||||
onChange={(v) => setNotifications((n) => ({ ...n, integrationFailureThreshold: v }))}
|
||||
/>
|
||||
<div className="mt-3">{toggle("automationAlerts", "Semaphore template or Gitea workflow run failed")}</div>
|
||||
<div className="mt-3">{toggle("updateRunResults", "An update run you started finished, or didn't")}</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Reference in new issue
Block a user