diff --git a/AGENT_UPDATE.md b/AGENT_UPDATE.md index c9b2d49..7d3e81a 100644 --- a/AGENT_UPDATE.md +++ b/AGENT_UPDATE.md @@ -215,13 +215,14 @@ The same three parts, from what Windows itself knows (the agent runs as SYSTEM, | **Disk health** | `Get-PhysicalDisk` and its reliability counters | Windows' own verdict: *Healthy* passes, *Warning* and *Unhealthy* fail, anything else says nothing. Temperature and wear come from the counters where the drive gives sane ones. Virtual disks are left out. | | **Storage pools** | `Get-StoragePool` (Storage Spaces) | *Healthy*, *Warning* and *Unhealthy* are reported as ONLINE, DEGRADED and FAULTED, with how full the pool is, and show beside ZFS pools. A machine with no pools reports an empty list. | -## Services to ignore, from the server's page +## Services to watch, from the server's page -A service that always shows as failed on one machine (an updater, a unit that is meant to be stopped) can be ignored there: on the server's **Host health** -card an administrator presses **Ignore** beside it, or types a name or a pattern with `*` (`gupdate*`, `snap.*`; case doesn't matter and `.service` is -optional). It stops counting as failed — no alert, not in `/metrics` — the moment it is saved, with no agent to update, and it still shows on the page, -marked *ignored*, with a button to stop ignoring it. The list belongs to that server (it is kept with its own alert limits), and works the same for Linux and -Windows; the Windows agent's own `ignoreServices` still works as well. +Services are **opt-in**. The agents report every service that has failed, but none of them raises an alert until you say it matters: on the server's **Host health** card an administrator +types a name or a pattern with `*` (`nginx`, `docker*`, `postgresql*`; case doesn't matter and `.service` is optional; `*` alone watches every service, which is how it +worked before), or presses **Watch** beside a service that has failed. A watched service that has failed raises the alert and counts in `/metrics`; one that is not watched still shows on the page, +marked *not watched*, with its **Watch** button. It takes effect the moment it is saved, with no agent to update. The list belongs to that server (it is kept with its own alert limits) and works +the same for Linux and Windows; the Windows agent's own `ignoreServices` (which leaves a service out of what the agent reports at all) still works as well. Only a service that has *failed* is seen: +one that was merely stopped on Linux is not reported by the agent. ## What the agent couldn't collect (Linux version 6, Windows version 5) diff --git a/DATABASE.md b/DATABASE.md index 5087930..34af91b 100644 --- a/DATABASE.md +++ b/DATABASE.md @@ -276,7 +276,7 @@ what it last reported. | `no_parent` | bool NOT NULL, default 0 | An admin said this server runs on nothing: no link, and no guess from its Proxmox node. Choosing a server by hand, or going back to automatic, clears it. | | `pending_updates` | text | JSON `{"manager":"apt","count":5,"security":2,"rebootRequired":false,"reportedAt":"…"}` from the agent; null until it has said. `security` is null when the package manager can't tell. | | `updates_pending_since` | text | ISO; when updates were first seen waiting (and none were before). Null when none are waiting. | -| `thresholds` | text | This server's own alert limits as JSON: `{"serverOfflineMinutes":…,"diskUsagePercent":…,"diskForecastDays":…,"backupStaleHours":…,"mounts":{"/data":{"percent":95,"ignore":true}},"ignoreServices":["gupdate*"]}`. `ignoreServices` are services never counted as failed here (names or patterns with `*`, up to 30). Anything left out follows Settings; null when it follows Settings entirely. Ranges are checked on the way in (offline 15–10080 min, disk 50–99 %, forecast 0–90 days, backup 0–2160 h; 0 means off). | +| `thresholds` | text | This server's own alert limits as JSON: `{"serverOfflineMinutes":…,"diskUsagePercent":…,"diskForecastDays":…,"backupStaleHours":…,"mounts":{"/data":{"percent":95,"ignore":true}},"watchServices":["nginx","docker*"]}`. `watchServices` are the services watched here (names or patterns with `*`, up to 50): only a watched service that has failed raises an alert or counts in `/metrics`; with none listed no service does. An old `ignoreServices` entry is no longer read. Anything left out follows Settings; null when it follows Settings entirely. Ranges are checked on the way in (offline 15–10080 min, disk 50–99 %, forecast 0–90 days, backup 0–2160 h; 0 means off). | | `host_health` | text | JSON `{"failedUnits":[{unit,description}],"smart":[{device,model,healthy,temperatureC,wearPercent}],"zfs":[{name,health,capacityPercent}],"reportedAt":"…"}`. Each part is present only if the agent reported it, and a report that leaves one out leaves it as it was. Null until an agent says anything. Judged by the health check and shown on the server's page. | | `hw_manufacturer`, `hw_model`, `hw_serial` | text | What the machine is, as its firmware says (the agent reports it; placeholders like "To Be Filled By O.E.M." are left out). Each is replaced only when a report has a value, so an agent that can't tell doesn't wipe what it said. Used to fill in the inventory. | | `agent_diagnostics` | text | JSON `{"items":[{check,status,reason?,detail?}],"reportedAt":"…"}`: what the agent says it could not collect (`status` is `unavailable` or `failed`). Replaced by each report that carries the list (an empty list means nothing was missed) and kept when a report doesn't. Turned into words and fixes by the server; never sent to the browser as it is. | diff --git a/README.md b/README.md index 1ff7695..cd9aa32 100644 --- a/README.md +++ b/README.md @@ -737,8 +737,8 @@ Proxmox guest) how old its backup may be — and per disk, either a limit or "no for this disk". Whatever is left empty follows Settings, so changing the global limit still reaches every server that hasn't said otherwise. They apply everywhere the global ones did: the alerts, the notifications, the dashboard, the map, the forecast, the -digest and `/metrics`. The same place keeps the **services to ignore** on that server — an updater that always shows as failed, say — which can also be set from its -Host health card; and a card, **What the agent couldn't collect**, tells you why a figure is missing (no `smartctl`, an update check that didn't answer …) and what +digest and `/metrics`. The same place keeps the **services to watch** on that server — only a service you list there raises an alert when it has failed, so the many that don't matter stay quiet; the list can also be set from its +Host health card (a failed service that isn't watched shows there with a **Watch** button; `*` watches them all); and a card, **What the agent couldn't collect**, tells you why a figure is missing (no `smartctl`, an update check that didn't answer …) and what to do. See AGENT_UPDATE.md. **Inventory** (Reference → Inventory) keeps the hardware behind it all: name and diff --git a/docs/MONITORING.md b/docs/MONITORING.md index 2bc6e57..5eef245 100644 --- a/docs/MONITORING.md +++ b/docs/MONITORING.md @@ -134,7 +134,7 @@ Every threshold here is a default; where it can be changed, the place is named. | **Server offline** | The agent has not reported for longer than the limit | 60 min · critical | Settings → Notifications; per server | | **Disk nearly full** | A server disk, a Proxmox node's root filesystem or storage, or a Synology volume is at or over the limit | 90% · warning, **critical from 95%** | Settings → Notifications; per server, per disk | | **Disk filling up** | A disk on a steady climb will be full within the window ([Forecasts](#forecasts)) | 14 days · warning, critical within 3 days | Settings → Notifications; per server | -| **Failed services** | A Linux agent reports failed systemd units | any | per server (ignore list) | +| **Failed services** | A server's agent reports failed services (systemd units on Linux, auto-start services that stopped with an error on Windows); only the ones the server is told to **watch** count | any | per server (watch list) | | **Disk health** | A disk's SMART self-assessment fails (critical) or an SSD has used ≥ 90% of its rated life (warning) | | not configurable | | **ZFS / storage pool** | A ZFS pool (or Windows Storage Spaces pool) is not ONLINE | | not configurable | | **Synology volume / disk** | A volume's status is not "normal"; a disk's status or SMART result is not normal, or it is over the bad-sector or under the remaining-life limit | | not configurable | diff --git a/server/src/routes/servers.ts b/server/src/routes/servers.ts index 32f4734..ed93427 100644 --- a/server/src/routes/servers.ts +++ b/server/src/routes/servers.ts @@ -8,7 +8,7 @@ import { generateApiToken } from "../services/tokens.js"; import { recordAudit } from "../services/audit.js"; import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js"; import { parsePendingUpdates } from "../services/pendingUpdates.js"; -import { markIgnoredServices, parseHostHealth } from "../services/hostHealth.js"; +import { markWatchedServices, parseHostHealth } from "../services/hostHealth.js"; import { describeDiagnostics, parseDiagnostics } from "../services/agentDiagnostics.js"; import { checkThresholds, parseThresholds } from "../services/serverThresholds.js"; import { latestAgentVersions } from "../services/agentVersions.js"; @@ -34,7 +34,7 @@ export const serversRouter = Router(); /** A server row as the API returns it: no token hash, and tags as an array rather than the stored JSON. */ function publicServer(row: T) { const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, thresholds, agentDiagnostics: _diagnostics, ...rest } = row; - return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth), thresholds: parseThresholds(thresholds) }; + return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: markWatchedServices(parseHostHealth(hostHealth), parseThresholds(thresholds)?.watchServices), thresholds: parseThresholds(thresholds) }; } serversRouter.use(requireAuth); @@ -452,7 +452,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => { }; const settings = await getSettings(); - res.json({ os: osSummary(server, new Date(), settings.healthChecks.osSupportWarnDays), server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: markIgnoredServices(parseHostHealth(rawHealth), parseThresholds(rawThresholds)?.ignoreServices), thresholds: parseThresholds(rawThresholds) }, diagnostics: describeDiagnostics(parseDiagnostics(rawDiagnostics), { osType: server.osType, lastSeenAt: server.lastSeenAt, agentVersion: server.agentVersion, latest: latestAgentVersions() }), thresholdDefaults: (await getSettings()).healthChecks, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() }); + res.json({ os: osSummary(server, new Date(), settings.healthChecks.osSupportWarnDays), server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: markWatchedServices(parseHostHealth(rawHealth), parseThresholds(rawThresholds)?.watchServices), thresholds: parseThresholds(rawThresholds) }, diagnostics: describeDiagnostics(parseDiagnostics(rawDiagnostics), { osType: server.osType, lastSeenAt: server.lastSeenAt, agentVersion: server.agentVersion, latest: latestAgentVersions() }), thresholdDefaults: (await getSettings()).healthChecks, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() }); })); const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) }); diff --git a/server/src/services/alertNoise.ts b/server/src/services/alertNoise.ts index 6770678..57b0c98 100644 --- a/server/src/services/alertNoise.ts +++ b/server/src/services/alertNoise.ts @@ -79,7 +79,7 @@ export function hintFor(category: string, alertId: string): string { if (key.startsWith("heartbeat")) return "If the job is just sometimes late, give its heartbeat more grace; if it fails, look at what it printed."; if (key.startsWith("endpoint:")) return "Raise “Alert after … failures in a row” on the check, or its timeout or interval — or fix what is making it fail."; if (key.startsWith("endpoint-cert:")) return "Renew the certificate, or lower how early the check warns."; - if (key.startsWith("failed-units:")) return "If a service is expected to fail, add it to “Services to ignore” on the server's Host health card; otherwise fix the service."; + if (key.startsWith("failed-units:")) return "If this service is not important, take it off the server's “Services to watch” on its Host health card; otherwise fix the service."; if (key.startsWith("disk-forecast:") || category === "disk") return "Raise the limit for this disk (a server can have its own, per disk), or leave it alone on the server's page if it is meant to be full."; if (category === "offline") return "If the server is just slow to report, raise “Server offline after” under Settings → Notifications, or give this server its own limit."; if (category === "integration") return "Raise “Alert after … consecutive failures” under Settings → Notifications; the Call log (Integrations) shows what is failing."; diff --git a/server/src/services/healthMonitor.ts b/server/src/services/healthMonitor.ts index 4519bf8..72b1037 100644 --- a/server/src/services/healthMonitor.ts +++ b/server/src/services/healthMonitor.ts @@ -11,7 +11,7 @@ import { isFresh, syncStatuses } from "./unifiSync.js"; import { activeSubjects, subjectOfConditionKey } from "./maintenance.js"; import { dependencyCover, HEALTH_STATE_FLAG } from "./dependencies.js"; import { getFailingSources } from "./integrationHealthMonitor.js"; -import { hostHealthConditions, parseHostHealth, withoutIgnoredServices, type HostHealth } from "./hostHealth.js"; +import { hostHealthConditions, parseHostHealth, onlyWatchedServices, type HostHealth } from "./hostHealth.js"; import { newListeningPorts, newPortMessage, parseBaseline, portKey, type NewPort } from "./newPorts.js"; import { listPresets, serviceName, serviceNames } from "./portPresets.js"; import { containerLabel, containerPortName, containerPortNames, purgeStaleContainerPorts, refreshContainerPorts } from "./containerPorts.js"; @@ -558,7 +558,7 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel // a malformed report shouldn't stop every other server being checked } const newPorts = checks.newPortAlerts ? newListeningPorts(parseJson(s.listeningPorts, null), parseBaseline(s.portBaseline), expectedByServer.get(s.id) ?? new Set()).map((p) => ({ ...p, service: containerLabel(containerPortName(containerNames, s.id, "tcp", p.port)) ?? serviceName(portNames, "tcp", p.port) })) : []; - snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: withoutIgnoredServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.ignoreServices), thresholds: parseThresholds(s.thresholds), newPorts }); + snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: onlyWatchedServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.watchServices), thresholds: parseThresholds(s.thresholds), newPorts }); } // Their own rules: what the stored history says. A failure here costs only the rules. diff --git a/server/src/services/hostHealth.ts b/server/src/services/hostHealth.ts index c08ba97..f7bcd7f 100644 --- a/server/src/services/hostHealth.ts +++ b/server/src/services/hostHealth.ts @@ -8,8 +8,8 @@ export interface FailedUnit { unit: string; description: string; - /** Set where it is shown, for a unit this server has been told to ignore. It never counts as failed. */ - ignored?: boolean; + /** Set where it is shown: whether this server is told to watch it. A unit that is not watched never counts as failed. */ + watched?: boolean; } export interface SmartDisk { device: string; @@ -86,8 +86,8 @@ export function nextHostHealth(previous: string | null, incoming: IncomingHostHe const globToRegex = (pattern: string) => new RegExp(`^${pattern.trim().toLowerCase().replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*")}$`); -/** Whether a unit is one this server is told to ignore: a name, or a pattern with * ("gupdate*"); case doesn't matter and ".service" may be left off. */ -export function isServiceIgnored(unit: string, patterns: string[] | undefined): boolean { +/** Whether a unit is one of those a server is told to watch: a name, or a pattern with * ("docker*"; "*" is every service); case doesn't matter and ".service" may be left off. */ +export function isServiceWatched(unit: string, patterns: string[] | undefined): boolean { if (!patterns || patterns.length === 0) return false; const u = unit.toLowerCase(); const bare = u.replace(/\.service$/, ""); @@ -97,16 +97,19 @@ export function isServiceIgnored(unit: string, patterns: string[] | undefined): }); } -/** The health with the ignored services taken out: what alerts and figures are worked from. */ -export function withoutIgnoredServices(health: HostHealth | null, patterns: string[] | undefined): HostHealth | null { - if (!health?.failedUnits || !patterns || patterns.length === 0) return health; - return { ...health, failedUnits: health.failedUnits.filter((u) => !isServiceIgnored(u.unit, patterns)) }; +/** + * The health with only the watched services kept: what alerts and figures are worked from. Services are opt-in — with nothing watched, no service counts as failed, however many the agent reports. + * A report with no part about services at all (an agent that could not collect it) stays as it is. + */ +export function onlyWatchedServices(health: HostHealth | null, patterns: string[] | undefined): HostHealth | null { + if (!health?.failedUnits) return health; + return { ...health, failedUnits: health.failedUnits.filter((u) => isServiceWatched(u.unit, patterns)) }; } -/** The health with the ignored services marked, not removed: what the server's page shows, so they can be seen and taken off the list. */ -export function markIgnoredServices(health: HostHealth | null, patterns: string[] | undefined): HostHealth | null { - if (!health?.failedUnits || !patterns || patterns.length === 0) return health; - return { ...health, failedUnits: health.failedUnits.map((u) => (isServiceIgnored(u.unit, patterns) ? { ...u, ignored: true } : u)) }; +/** The health with each failed service marked as watched or not, none removed: what the server's page shows, so a failed service can be seen and picked to be watched. */ +export function markWatchedServices(health: HostHealth | null, patterns: string[] | undefined): HostHealth | null { + if (!health?.failedUnits) return health; + return { ...health, failedUnits: health.failedUnits.map((u) => ({ ...u, watched: isServiceWatched(u.unit, patterns) })) }; } /** A pool in any state but ONLINE has lost something (DEGRADED has lost redundancy, the rest are worse). */ diff --git a/server/src/services/metrics.ts b/server/src/services/metrics.ts index 96b1ea6..ef768fa 100644 --- a/server/src/services/metrics.ts +++ b/server/src/services/metrics.ts @@ -36,7 +36,7 @@ import { getAlerts } from "./alerts.js"; import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js"; import { getSourceStatus } from "./integrationHealthMonitor.js"; import { parsePendingUpdates } from "./pendingUpdates.js"; -import { parseHostHealth, poolIsFine, withoutIgnoredServices } from "./hostHealth.js"; +import { parseHostHealth, poolIsFine, onlyWatchedServices } from "./hostHealth.js"; import { parseThresholds as parseServerThresholds } from "./serverThresholds.js"; import { parseThresholds } from "./serverThresholds.js"; import { getCachedBackupCoverage } from "./backupCoverage.js"; @@ -534,7 +534,7 @@ export async function buildMetrics(now: Date = new Date()): Promise { if (updates.security !== null) out.add("homelab_server_pending_security_updates", "gauge", "Of those, the ones that are security fixes.", updates.security, { server }); out.add("homelab_server_reboot_required", "gauge", "1 if a restart is due.", updates.rebootRequired ? 1 : 0, { server }); } - const health = withoutIgnoredServices(parseHostHealth(s.hostHealth), parseServerThresholds(s.thresholds)?.ignoreServices); + const health = onlyWatchedServices(parseHostHealth(s.hostHealth), parseServerThresholds(s.thresholds)?.watchServices); if (health?.failedUnits) out.add("homelab_server_failed_units", "gauge", "Services (systemd units) that have failed on the server.", health.failedUnits.length, { server }); for (const d of health?.smart ?? []) { if (d.healthy !== null) out.add("homelab_server_disk_healthy", "gauge", "1 if the disk's own SMART self-assessment passes, 0 if it fails.", d.healthy ? 1 : 0, { server, device: d.device }); diff --git a/server/src/services/publicApi.ts b/server/src/services/publicApi.ts index abe80cd..828677c 100644 --- a/server/src/services/publicApi.ts +++ b/server/src/services/publicApi.ts @@ -10,7 +10,7 @@ import { attentionCounts, loadAcks } from "./alertAcks.js"; import { getAlerts } from "./alerts.js"; import { loadStats } from "./endpointChecks.js"; import { describeTarget, listActiveWindows, listUpcomingWindows } from "./maintenance.js"; -import { parseHostHealth } from "./hostHealth.js"; +import { onlyWatchedServices, parseHostHealth } from "./hostHealth.js"; import { parsePendingUpdates } from "./pendingUpdates.js"; import { parseTags } from "./serverTags.js"; import { parseThresholds } from "./serverThresholds.js"; @@ -38,7 +38,7 @@ export function serverV1(row: ServerRow, offlineMinutes: number, now: Date, osWa const status = serverStatus(row, offlineMinutes, now); const disks = json<{ mount: string; sizeBytes: number; usedBytes: number }[]>(row.disks, []); const updates = parsePendingUpdates(row.pendingUpdates); - const health = parseHostHealth(row.hostHealth); + const health = onlyWatchedServices(parseHostHealth(row.hostHealth), parseThresholds(row.thresholds)?.watchServices); return { id: row.id, name: row.name, diff --git a/server/src/services/serverThresholds.ts b/server/src/services/serverThresholds.ts index d55fa4c..42735b6 100644 --- a/server/src/services/serverThresholds.ts +++ b/server/src/services/serverThresholds.ts @@ -21,8 +21,8 @@ export interface ServerThresholds { /** 0 turns the stale-backup alert off for this server's guest. */ backupStaleHours?: number; mounts?: Record; - /** Services (units) never to count as failed on this server: names or patterns like "gupdate*" (case does not matter, ".service" is optional). */ - ignoreServices?: string[]; + /** The services (units) to watch on this server: names or patterns like "docker*" (case does not matter, ".service" is optional; "*" is all of them). Only a watched service that has failed raises an alert. */ + watchServices?: string[]; } /** The global values these override. */ @@ -40,7 +40,7 @@ export const LIMITS = { backupStaleHours: { min: 0, max: 2160 }, } as const; export const MAX_MOUNT_RULES = 50; -export const MAX_IGNORED_SERVICES = 30; +export const MAX_WATCHED_SERVICES = 50; const MAX_SERVICE_PATTERN = 100; const inRange = (v: unknown, key: keyof typeof LIMITS): v is number => typeof v === "number" && Number.isInteger(v) && v >= LIMITS[key].min && v <= LIMITS[key].max; @@ -65,10 +65,10 @@ export function parseThresholds(stored: string | null | undefined): ServerThresh } if (Object.keys(mounts).length > 0) out.mounts = mounts; } - if (Array.isArray(v.ignoreServices)) { - const strings: string[] = (v.ignoreServices as unknown[]).filter((p): p is string => typeof p === "string"); - const list = [...new Set(strings.map((p) => p.trim()).filter((p) => p.length > 0 && p.length <= MAX_SERVICE_PATTERN))].slice(0, MAX_IGNORED_SERVICES); - if (list.length > 0) out.ignoreServices = list; + if (Array.isArray(v.watchServices)) { + const strings: string[] = (v.watchServices as unknown[]).filter((p): p is string => typeof p === "string"); + const list = [...new Set(strings.map((p) => p.trim()).filter((p) => p.length > 0 && p.length <= MAX_SERVICE_PATTERN))].slice(0, MAX_WATCHED_SERVICES); + if (list.length > 0) out.watchServices = list; } return Object.keys(out).length > 0 ? out : null; } catch { @@ -112,19 +112,19 @@ export function checkThresholds(input: unknown): { ok: true; value: ServerThresh } if (Object.keys(mounts).length > 0) out.mounts = mounts; } - if (v.ignoreServices !== undefined && v.ignoreServices !== null) { - if (!Array.isArray(v.ignoreServices)) return { ok: false, message: "The services to ignore should be a list." }; + if (v.watchServices !== undefined && v.watchServices !== null) { + if (!Array.isArray(v.watchServices)) return { ok: false, message: "The services to watch should be a list." }; const list: string[] = []; - for (const raw of v.ignoreServices) { - if (typeof raw !== "string") return { ok: false, message: "Each service to ignore should be a name or a pattern." }; + for (const raw of v.watchServices) { + if (typeof raw !== "string") return { ok: false, message: "Each service to watch should be a name or a pattern." }; const p = raw.trim(); if (p.length === 0) continue; // eslint-disable-next-line no-control-regex if (p.length > MAX_SERVICE_PATTERN || /[\u0000-\u001f]/.test(p)) return { ok: false, message: `"${p.slice(0, 30)}…" isn't a service name or pattern (up to ${MAX_SERVICE_PATTERN} characters, no line breaks).` }; if (!list.includes(p)) list.push(p); } - if (list.length > MAX_IGNORED_SERVICES) return { ok: false, message: `At most ${MAX_IGNORED_SERVICES} services can be ignored on one server.` }; - if (list.length > 0) out.ignoreServices = list; + if (list.length > MAX_WATCHED_SERVICES) return { ok: false, message: `At most ${MAX_WATCHED_SERVICES} services can be watched on one server.` }; + if (list.length > 0) out.watchServices = list; } return { ok: true, value: Object.keys(out).length > 0 ? out : null }; } diff --git a/server/test/agentDiagnostics.test.ts b/server/test/agentDiagnostics.test.ts index ebd893b..da9002e 100644 --- a/server/test/agentDiagnostics.test.ts +++ b/server/test/agentDiagnostics.test.ts @@ -22,7 +22,7 @@ const { db } = await import("../src/db/client.js"); const { users, servers } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { describeDiagnostics, parseDiagnostics } = await import("../src/services/agentDiagnostics.js"); -const { isServiceIgnored, withoutIgnoredServices, markIgnoredServices } = await import("../src/services/hostHealth.js"); +const { isServiceWatched, onlyWatchedServices, markWatchedServices } = await import("../src/services/hostHealth.js"); const { checkThresholds, parseThresholds } = await import("../src/services/serverThresholds.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { buildMetrics } = await import("../src/services/metrics.js"); @@ -89,36 +89,44 @@ describe("saying what to do about what couldn't be collected", () => { }); }); -describe("services a server is told to ignore", () => { +describe("services a server is told to watch", () => { test("names and patterns, in any case, with or without .service", () => { - assert.equal(isServiceIgnored("gupdate.service", ["gupdate"]), true); - assert.equal(isServiceIgnored("GUpdate", ["gupdate*"]), true); - assert.equal(isServiceIgnored("snap.lxd.activate.service", ["snap.*"]), true); - assert.equal(isServiceIgnored("nginx.service", ["gupdate*", "snap.*"]), false); - assert.equal(isServiceIgnored("nginx.service", ["nginx.service"]), true); - assert.equal(isServiceIgnored("mynginx.service", ["nginx"]), false, "a name is the whole name"); - assert.equal(isServiceIgnored("a+b.service", ["a+b"]), true, "characters that mean something to a regex are only characters here"); - assert.equal(isServiceIgnored("anything", []), false); - assert.equal(isServiceIgnored("anything", undefined), false); + assert.equal(isServiceWatched("gupdate.service", ["gupdate"]), true); + assert.equal(isServiceWatched("GUpdate", ["gupdate*"]), true); + assert.equal(isServiceWatched("snap.lxd.activate.service", ["snap.*"]), true); + assert.equal(isServiceWatched("nginx.service", ["gupdate*", "snap.*"]), false); + assert.equal(isServiceWatched("nginx.service", ["nginx.service"]), true); + assert.equal(isServiceWatched("mynginx.service", ["nginx"]), false, "a name is the whole name"); + assert.equal(isServiceWatched("a+b.service", ["a+b"]), true, "characters that mean something to a regex are only characters here"); + assert.equal(isServiceWatched("anything", []), false); + assert.equal(isServiceWatched("anything", undefined), false); }); - test("left out for what is judged, marked for what is shown", () => { + test("* is every service", () => { + assert.equal(isServiceWatched("anything.service", ["*"]), true); + }); + + test("only the watched are kept for what is judged, all are shown marked", () => { const health = { failedUnits: [{ unit: "nginx.service", description: "" }, { unit: "gupdate", description: "Google Update" }], reportedAt: "t" }; - assert.deepEqual(withoutIgnoredServices(health, ["gupdate*"])!.failedUnits!.map((u) => u.unit), ["nginx.service"]); - assert.deepEqual(markIgnoredServices(health, ["gupdate*"])!.failedUnits!.map((u) => [u.unit, u.ignored ?? false]), [["nginx.service", false], ["gupdate", true]]); - assert.equal(withoutIgnoredServices(null, ["x"]), null); - assert.deepEqual(withoutIgnoredServices(health, []), health); + assert.deepEqual(onlyWatchedServices(health, ["nginx"])!.failedUnits!.map((u) => u.unit), ["nginx.service"]); + assert.deepEqual(markWatchedServices(health, ["nginx"])!.failedUnits!.map((u) => [u.unit, u.watched]), [["nginx.service", true], ["gupdate", false]]); + assert.equal(onlyWatchedServices(null, ["x"]), null); + assert.deepEqual(onlyWatchedServices(health, [])!.failedUnits, [], "nothing is watched until it is said: no service counts as failed"); + assert.deepEqual(onlyWatchedServices(health, undefined)!.failedUnits, []); + assert.deepEqual(markWatchedServices(health, [])!.failedUnits!.map((u) => u.watched), [false, false]); + const silent = { reportedAt: "t" }; + assert.deepEqual(onlyWatchedServices(silent, []), silent, "a report with nothing about services is left as it is"); }); test("the list is part of a server's own thresholds: kept tidy, and a wrong one is refused", () => { - const ok = checkThresholds({ ignoreServices: [" gupdate* ", "gupdate*", "", "snap.*"] }); - assert.deepEqual(ok.ok && ok.value, { ignoreServices: ["gupdate*", "snap.*"] }); - assert.equal(checkThresholds({ ignoreServices: "gupdate" }).ok, false); - assert.equal(checkThresholds({ ignoreServices: [3] }).ok, false); - assert.equal(checkThresholds({ ignoreServices: ["x".repeat(101)] }).ok, false); - assert.equal(checkThresholds({ ignoreServices: Array.from({ length: 31 }, (_, i) => `s${i}`) }).ok, false); - assert.equal(checkThresholds({ ignoreServices: [] }).ok && (checkThresholds({ ignoreServices: [] }) as { value: unknown }).value, null, "an empty list is no thresholds at all"); - assert.deepEqual(parseThresholds(JSON.stringify({ ignoreServices: ["a", 3, "", "a"] })), { ignoreServices: ["a"] }); + const ok = checkThresholds({ watchServices: [" gupdate* ", "gupdate*", "", "snap.*"] }); + assert.deepEqual(ok.ok && ok.value, { watchServices: ["gupdate*", "snap.*"] }); + assert.equal(checkThresholds({ watchServices: "gupdate" }).ok, false); + assert.equal(checkThresholds({ watchServices: [3] }).ok, false); + assert.equal(checkThresholds({ watchServices: ["x".repeat(101)] }).ok, false); + assert.equal(checkThresholds({ watchServices: Array.from({ length: 51 }, (_, i) => `s${i}`) }).ok, false); + assert.equal(checkThresholds({ watchServices: [] }).ok && (checkThresholds({ watchServices: [] }) as { value: unknown }).value, null, "an empty list is no thresholds at all"); + assert.deepEqual(parseThresholds(JSON.stringify({ watchServices: ["a", 3, "", "a"] })), { watchServices: ["a"] }); }); }); @@ -179,28 +187,29 @@ describe("through a report, the page, the alerts and the figures", () => { assert.equal(row.agentDiagnostics, undefined, "never as raw stored text"); }); - test("a service the server is told to ignore stops being an alert and a figure, still shows on the page marked, and can be un-ignored", async () => { + test("nothing is an alert until it is watched; a watched service that has failed is, and a failed one that is not watched still shows on the page", async () => { assert.equal((await report({ health: { failed_units: [{ unit: "nginx.service", description: "web" }, { unit: "gupdate.service", description: "Google Update" }] } })).status, 202); invalidateAlertsCache(); const failing = async () => (await getAlerts(true)).alerts.find((a) => a.id.startsWith("services:failed-units") && a.id.includes(`:server:${id}`)); - assert.match((await failing())!.message, /2 services have failed/); + assert.equal(await failing(), undefined, "nothing is watched, so nothing is an alert"); + assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\{server="elsa"[^}]*\} 0`)); + assert.deepEqual((await detail()).server.hostHealth.failedUnits.map((u: { unit: string; watched?: boolean }) => [u.unit, u.watched]), [["nginx.service", false], ["gupdate.service", false]], "still shown, marked not watched"); - assert.equal((await patch({ thresholds: { ignoreServices: ["gupdate*"] } })).status, 200); + assert.equal((await patch({ thresholds: { watchServices: ["nginx"] } })).status, 200); assert.match((await failing())!.message, /1 service has failed — nginx\.service/, "the alert cache was dropped when the list changed"); - assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\\{server="elsa"[^}]*\\} 1`)); - + assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\{server="elsa"[^}]*\} 1`)); const units = (await detail()).server.hostHealth.failedUnits; - assert.deepEqual(units.map((u: { unit: string; ignored?: boolean }) => [u.unit, u.ignored ?? false]), [["nginx.service", false], ["gupdate.service", true]]); + assert.deepEqual(units.map((u: { unit: string; watched?: boolean }) => [u.unit, u.watched]), [["nginx.service", true], ["gupdate.service", false]]); - assert.equal((await patch({ thresholds: { ignoreServices: ["gupdate*", "nginx"] } })).status, 200); - assert.equal(await failing(), undefined, "with every failed service ignored there is nothing to alert about"); + assert.equal((await patch({ thresholds: { watchServices: ["*"] } })).status, 200); + assert.match((await failing())!.message, /2 services have failed/, "* watches them all, as it always was"); assert.equal((await patch({ thresholds: null })).status, 200); - assert.match((await failing())!.message, /2 services have failed/); + assert.equal(await failing(), undefined, "taking the list away watches nothing again"); }); test("a list that isn't one is refused with a message", async () => { - const res = await patch({ thresholds: { ignoreServices: "gupdate" } }); + const res = await patch({ thresholds: { watchServices: "gupdate" } }); assert.equal(res.status, 400); assert.match((await res.json()).message, /should be a list/); }); diff --git a/server/test/flapping.test.ts b/server/test/flapping.test.ts index 5ad2a7d..9b26fa4 100644 --- a/server/test/flapping.test.ts +++ b/server/test/flapping.test.ts @@ -273,7 +273,7 @@ describe("the noise report", () => { assert.match(hintFor("monitoring", "monitoring:new-port:server:1:tcp:8080"), /Ports card/); assert.match(hintFor("automation", "automation:heartbeat:1"), /grace/); assert.match(hintFor("monitoring", "monitoring:endpoint:2"), /failures in a row/); - assert.match(hintFor("services", "services:failed-units:server:1"), /Services to ignore/); + assert.match(hintFor("services", "services:failed-units:server:1"), /Services to watch/); assert.match(hintFor("tickets", "tickets:x"), /Acknowledge it/); }); diff --git a/server/test/hostHealthAndAgentUpdate.test.ts b/server/test/hostHealthAndAgentUpdate.test.ts index 160a80c..a841a65 100644 --- a/server/test/hostHealthAndAgentUpdate.test.ts +++ b/server/test/hostHealthAndAgentUpdate.test.ts @@ -109,7 +109,7 @@ describe("through the report, the alerts, the figures and the page", () => { { oidcSub: "admin", name: "Ada", role: "admin" }, { oidcSub: "operator", name: "Olle", role: "operator" }, ]); - const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t" }).returning(); + const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", thresholds: JSON.stringify({ watchServices: ["*"] }) }).returning(); id = s.id; const a = express(); a.use("/api/agent/report", agentReportRouter); diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 6947bc0..178fa5a 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -840,7 +840,7 @@ export interface AgentLatest { /** What a Linux agent says about the machine itself. Each part is there only if the agent could find it out. */ export interface HostHealth { - failedUnits?: { unit: string; description: string; /** This server is told to ignore it: it doesn't count as failed. */ ignored?: boolean }[]; + failedUnits?: { unit: string; description: string; /** Whether this server is told to watch it: one that is not watched does not count as failed. */ watched?: boolean }[]; smart?: { device: string; model: string; healthy: boolean | null; temperatureC: number | null; wearPercent: number | null }[]; zfs?: { name: string; health: string; capacityPercent: number | null }[]; reportedAt: string; @@ -863,8 +863,8 @@ export interface ServerThresholds { diskForecastDays?: number; backupStaleHours?: number; mounts?: Record; - /** Services never to count as failed here: names or patterns like "gupdate*". */ - ignoreServices?: string[]; + /** The services to watch here: names or patterns like "docker*" ("*" is all). Only a watched service that has failed raises an alert. */ + watchServices?: string[]; } /** One thing worth telling a person about a server's agent: what it couldn't collect (and what to do), or that it is behind. */ diff --git a/web/src/components/HostHealthCard.tsx b/web/src/components/HostHealthCard.tsx index 6498c33..7126cb0 100644 --- a/web/src/components/HostHealthCard.tsx +++ b/web/src/components/HostHealthCard.tsx @@ -13,25 +13,25 @@ const poolFine = (health: string) => health.toUpperCase() === "ONLINE"; */ export default function HostHealthCard({ health, - ignored = [], - onIgnoredChange, + watched = [], + onWatchedChange, }: { health: HostHealth | null; - /** The services this server is told to ignore (names or patterns). */ - ignored?: string[]; + /** The services this server is told to watch (names or patterns). Only these raise an alert when they fail. */ + watched?: string[]; /** Given when the person can change that list; it is saved by whoever holds it. */ - onIgnoredChange?: (list: string[]) => Promise | void; + onWatchedChange?: (list: string[]) => Promise | void; }) { const [pattern, setPattern] = useState(""); const [busy, setBusy] = useState(false); - if (!health && ignored.length === 0) return null; + if (!health && watched.length === 0) return null; const { failedUnits, smart, zfs } = health ?? { failedUnits: undefined, smart: undefined, zfs: undefined, reportedAt: "" }; async function change(list: string[]) { - if (!onIgnoredChange) return; + if (!onWatchedChange) return; setBusy(true); try { - await onIgnoredChange(list); + await onWatchedChange(list); setPattern(""); } finally { setBusy(false); @@ -39,7 +39,7 @@ export default function HostHealthCard({ } const hasSmart = !!smart && smart.length > 0; const hasZfs = !!zfs && zfs.length > 0; - if (failedUnits === undefined && !hasSmart && !hasZfs && ignored.length === 0) return null; + if (failedUnits === undefined && !hasSmart && !hasZfs && watched.length === 0) return null; return (
@@ -47,7 +47,7 @@ export default function HostHealthCard({

{t("Host health")}

{health?.reportedAt && (
- reported + {t("reported")}
)}
@@ -55,19 +55,18 @@ export default function HostHealthCard({ {failedUnits !== undefined && (
{t("Services")}
- {failedUnits.filter((u) => !u.ignored).length === 0 ? ( - none have failed - ) : null} + {failedUnits.length === 0 ? {t("none have failed")} : null} + {failedUnits.length > 0 && failedUnits.every((u) => u.watched === false) &&
{t("None of these is watched, so none raises an alert.")}
} {failedUnits.length > 0 && (
    {failedUnits.map((u) => ( -
  • - {u.ignored ? ignored : failed} +
  • + {u.watched === false ? {t("not watched")} : {t("failed")}} {u.unit} {u.description && {u.description}} - {onIgnoredChange && !u.ignored && ( - )}
  • @@ -77,32 +76,30 @@ export default function HostHealthCard({
)} - {(ignored.length > 0 || onIgnoredChange) && ( + {(watched.length > 0 || onWatchedChange) && (
-
{t("Services this server ignores")}
- {ignored.length === 0 &&
None. A service that always fails here (an updater, say) can be ignored, so it stops raising an alert.
} +
{t("Services to watch")}
+ {watched.length === 0 &&
{t("None. A service raises an alert only when it is listed here and has failed. Use * to watch them all.")}
}
- {ignored.map((p) => ( - + {watched.map((p) => ( + {p} - {onIgnoredChange && ( -
- {onIgnoredChange && ( + {onWatchedChange && (
{ e.preventDefault(); const p = pattern.trim(); - if (p && !ignored.includes(p)) void change([...ignored, p]); + if (p && !watched.includes(p)) void change([...watched, p]); }} > - setPattern(e.target.value)} /> + setPattern(e.target.value)} />
)} diff --git a/web/src/components/ServerThresholds.tsx b/web/src/components/ServerThresholds.tsx index fe77449..53d4247 100644 --- a/web/src/components/ServerThresholds.tsx +++ b/web/src/components/ServerThresholds.tsx @@ -14,7 +14,7 @@ const FIELDS: { key: Field; label: string; unit: string; min: number; max: numbe { key: "backupStaleHours", get label() { return t("Its backup is too old after"); }, unit: "hours", min: 0, max: 2160, hint: "For the Proxmox guest this server is. 0 means it is never called stale.", onlyForGuests: true }, ]; -const has = (t: Thresholds | null) => !!t && (Object.keys(t).some((k) => k !== "mounts" && k !== "ignoreServices") || Object.keys(t.mounts ?? {}).length > 0); +const has = (t: Thresholds | null) => !!t && (Object.keys(t).some((k) => k !== "mounts" && k !== "watchServices") || Object.keys(t.mounts ?? {}).length > 0); /** * This server's own alert limits. Anything left empty follows Settings → Notifications, so changing the global limit still reaches every @@ -55,7 +55,7 @@ export default function ServerThresholds({ server, defaults, disks, canEdit, onC } if (Object.keys(rules).length > 0) out.mounts = rules; // The services to ignore are edited on the health card; saving limits here must not lose them. - if (own?.ignoreServices?.length) out.ignoreServices = own.ignoreServices; + if (own?.watchServices?.length) out.watchServices = own.watchServices; return Object.keys(out).length > 0 ? out : null; } @@ -198,7 +198,7 @@ export default function ServerThresholds({ server, defaults, disks, canEdit, onC {busy ? "Saving…" : t("Save")} {has(own) && ( - )} diff --git a/web/src/i18n/sv.ts b/web/src/i18n/sv.ts index 0eecdb8..b16826d 100644 --- a/web/src/i18n/sv.ts +++ b/web/src/i18n/sv.ts @@ -84,6 +84,7 @@ export const sv: Record = { "A job to add": "Ett jobb att lägga till", "A month": "En månad", "A name like Europe/Stockholm, so 02:00 stays 02:00 in summer and winter.": "Ett namn som Europe/Stockholm, så att 02:00 förblir 02:00 sommar som vinter.", + "a name, or a pattern like docker*": "ett namn, eller ett mönster som docker*", "A page to open (optional)": "En sida att öppna (valfritt)", "A plan updates a group of servers on a schedule, one at a time, and stops if one fails. Tag the servers (for example “lab”), then make a plan for that tag.": "En plan uppdaterar en grupp servrar enligt schema, en i taget, och stannar om en misslyckas. Tagga servrarna (till exempel ”lab”) och skapa sedan en plan för taggen.", "A playbook that only restarts the server, in the same Semaphore. It runs once a server's updates are in; a server with nothing waiting that still says a restart is due is restarted too. Each restart is checked: the server must report that it started again. The window below counts for the update and the restart together.": "En playbook som bara startar om servern, i samma Semaphore. Den körs när en servers uppdateringar är installerade; en server utan väntande uppdateringar som ändå säger att en omstart behövs startas också om. Varje omstart kontrolleras: servern måste rapportera att den startade om. Fönstret nedan gäller för uppdateringen och omstarten tillsammans.", @@ -94,6 +95,7 @@ export const sv: Record = { "A scan from this app connected to it": "En skanning från den här appen kopplade upp sig mot den", "A server": "En server", "A service name or pattern to ignore": "Ett tjänstenamn eller mönster att ignorera", + "A service name or pattern to watch": "Ett tjänstenamn eller mönster att bevaka", "a week": "en vecka", "About": "Gäller", "About you": "Om dig", @@ -683,6 +685,7 @@ export const sv: Record = { "Export CSV": "Exportera CSV", "Exporting…": "Exporterar…", "External port": "Extern port", + "failed": "fallerade", "Failed": "Misslyckades", "Failed in the last day": "Misslyckades det senaste dygnet", "Failed to import": "Importen misslyckades", @@ -1275,10 +1278,13 @@ export const sv: Record = { "None connected.": "Ingen ansluten.", "None has been made.": "Ingen har gjorts.", "None has been taken.": "Ingen har tagits.", + "none have failed": "ingen har fallerat", + "None of these is watched, so none raises an alert.": "Ingen av dessa bevakas, så ingen ger larm.", "None to choose from": "Inga att välja bland", "None yet — add a picture of the rack or of a label.": "Inga ännu — lägg till en bild på racket eller på en etikett.", "None yet.": "Inga ännu.", "None.": "Inga.", + "None. A service raises an alert only when it is listed here and has failed. Use * to watch them all.": "Inga. En tjänst ger bara larm när den står här och har fallerat. Använd * för att bevaka alla.", "Normal": "Normal", "Not a VM? Hide this": "Ingen VM? Dölj det här", "Not answering": "Svarar inte", @@ -1308,6 +1314,7 @@ export const sv: Record = { "Not set up yet": "Inte uppsatt än", "Not silenced": "Inte tystad", "Not that": "Inte det", + "not watched": "bevakas inte", "Not working": "Fungerar inte", "Note": "Anteckning", "Note (optional)": "Anteckning (valfri)", @@ -1553,6 +1560,7 @@ export const sv: Record = { "Rack unit": "Rackenhet", "Racks": "Rack", "Raise alerts about it again": "Larma om den igen", + "Raise an alert when this service has failed on this server": "Larma när den här tjänsten har fallerat på den här servern", "Raised": "Utlöst", "Raised and cleared again and again within an hour — its messages are held back until it has been calm for an hour": "Larmat och släckts om och om igen inom en timme — dess meddelanden hålls tillbaka tills det har varit lugnt i en timme", "Raised and cleared often enough within an hour to count as flapping": "Larmat och släckts tillräckligt ofta inom en timme för att räknas som fladdrande", @@ -1633,6 +1641,7 @@ export const sv: Record = { "Replace names somebody wrote": "Ersätt namn som någon skrivit", "Replace this list": "Ersätt den här listan", "Replacement": "Ersättning", + "reported": "rapporterat", "Reported": "Rapporterad", "Reported by the agent as of": "Rapporterat av agenten per", "Reporting": "Rapporterar", @@ -1834,6 +1843,7 @@ export const sv: Record = { "Service levels": "Servicenivåer", "Services": "Tjänster", "Services this server ignores": "Tjänster som den här servern ignorerar", + "Services to watch": "Tjänster att bevaka", "Sessions": "Sessioner", "Set a target": "Sätt ett mål", "Set elsewhere": "Satt någon annanstans", @@ -1962,6 +1972,7 @@ export const sv: Record = { "Stop tracking": "Sluta spåra", "Stop tracking domain": "Sluta spåra domän", "Stop warning alerts": "Sluta ge varningslarm", + "Stop watching": "Sluta bevaka", "Stopped": "Stoppad", "stopped being followed": "följs inte längre", "stopping": "stoppar", @@ -2268,6 +2279,7 @@ export const sv: Record = { "Watch": "Bevaka", "Watch again": "Bevaka igen", "Watch something": "Bevaka något", + "Watched. It raises an alert if it fails on this server.": "Bevakas. Den ger larm om den slutar fungera på den här servern.", "Watching": "Bevakar", "Web address": "Webbadress", "Webhook delivered successfully.": "Webhooken levererades.", diff --git a/web/src/pages/ServerDetail.tsx b/web/src/pages/ServerDetail.tsx index 7cbdfc1..06de49b 100644 --- a/web/src/pages/ServerDetail.tsx +++ b/web/src/pages/ServerDetail.tsx @@ -826,14 +826,14 @@ export default function ServerDetail({ user }: { user: CurrentUser }) { <> { try { // The list is part of the server's own limits; the rest of them are carried over as they are. - await api.servers.update(server.id, { thresholds: { ...(server.thresholds ?? {}), ignoreServices: list.length > 0 ? list : undefined } }); - toast.success(list.length > (server.thresholds?.ignoreServices?.length ?? 0) ? "Ignored. It no longer counts as failed on this server." : "Saved."); + await api.servers.update(server.id, { thresholds: { ...(server.thresholds ?? {}), watchServices: list.length > 0 ? list : undefined } }); + toast.success(list.length > (server.thresholds?.watchServices?.length ?? 0) ? t("Watched. It raises an alert if it fails on this server.") : t("Saved.")); void loadDetail(); } catch (err) { toast.error(readableError(err));