Services: opt-in monitoring with a per-server watch list
A failed service now raises an alert (and counts in /metrics and the public API) only when the server is told to watch it, by name or pattern, or * for all. Failed services that are not watched still show on the Host health card with a Watch button. Replaces the ignore list. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6e39430a1f
commit
06ac120a1d
19 files changed
+143
-121
No files matched your search
@@ -8,7 +8,7 @@ import { generateApiToken } from "../services/tokens.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js";
|
||||
import { parsePendingUpdates } from "../services/pendingUpdates.js";
|
||||
import { markIgnoredServices, parseHostHealth } from "../services/hostHealth.js";
|
||||
import { markWatchedServices, parseHostHealth } from "../services/hostHealth.js";
|
||||
import { describeDiagnostics, parseDiagnostics } from "../services/agentDiagnostics.js";
|
||||
import { checkThresholds, parseThresholds } from "../services/serverThresholds.js";
|
||||
import { latestAgentVersions } from "../services/agentVersions.js";
|
||||
@@ -34,7 +34,7 @@ export const serversRouter = Router();
|
||||
/** A server row as the API returns it: no token hash, and tags as an array rather than the stored JSON. */
|
||||
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null; hostHealth: string | null; thresholds: string | null; agentDiagnostics?: string | null }>(row: T) {
|
||||
const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, thresholds, agentDiagnostics: _diagnostics, ...rest } = row;
|
||||
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth), thresholds: parseThresholds(thresholds) };
|
||||
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: markWatchedServices(parseHostHealth(hostHealth), parseThresholds(thresholds)?.watchServices), thresholds: parseThresholds(thresholds) };
|
||||
}
|
||||
|
||||
serversRouter.use(requireAuth);
|
||||
@@ -452,7 +452,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
|
||||
};
|
||||
|
||||
const settings = await getSettings();
|
||||
res.json({ os: osSummary(server, new Date(), settings.healthChecks.osSupportWarnDays), server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: markIgnoredServices(parseHostHealth(rawHealth), parseThresholds(rawThresholds)?.ignoreServices), thresholds: parseThresholds(rawThresholds) }, diagnostics: describeDiagnostics(parseDiagnostics(rawDiagnostics), { osType: server.osType, lastSeenAt: server.lastSeenAt, agentVersion: server.agentVersion, latest: latestAgentVersions() }), thresholdDefaults: (await getSettings()).healthChecks, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
|
||||
res.json({ os: osSummary(server, new Date(), settings.healthChecks.osSupportWarnDays), server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: markWatchedServices(parseHostHealth(rawHealth), parseThresholds(rawThresholds)?.watchServices), thresholds: parseThresholds(rawThresholds) }, diagnostics: describeDiagnostics(parseDiagnostics(rawDiagnostics), { osType: server.osType, lastSeenAt: server.lastSeenAt, agentVersion: server.agentVersion, latest: latestAgentVersions() }), thresholdDefaults: (await getSettings()).healthChecks, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
|
||||
}));
|
||||
|
||||
const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) });
|
||||
|
||||
@@ -79,7 +79,7 @@ export function hintFor(category: string, alertId: string): string {
|
||||
if (key.startsWith("heartbeat")) return "If the job is just sometimes late, give its heartbeat more grace; if it fails, look at what it printed.";
|
||||
if (key.startsWith("endpoint:")) return "Raise “Alert after … failures in a row” on the check, or its timeout or interval — or fix what is making it fail.";
|
||||
if (key.startsWith("endpoint-cert:")) return "Renew the certificate, or lower how early the check warns.";
|
||||
if (key.startsWith("failed-units:")) return "If a service is expected to fail, add it to “Services to ignore” on the server's Host health card; otherwise fix the service.";
|
||||
if (key.startsWith("failed-units:")) return "If this service is not important, take it off the server's “Services to watch” on its Host health card; otherwise fix the service.";
|
||||
if (key.startsWith("disk-forecast:") || category === "disk") return "Raise the limit for this disk (a server can have its own, per disk), or leave it alone on the server's page if it is meant to be full.";
|
||||
if (category === "offline") return "If the server is just slow to report, raise “Server offline after” under Settings → Notifications, or give this server its own limit.";
|
||||
if (category === "integration") return "Raise “Alert after … consecutive failures” under Settings → Notifications; the Call log (Integrations) shows what is failing.";
|
||||
|
||||
@@ -11,7 +11,7 @@ import { isFresh, syncStatuses } from "./unifiSync.js";
|
||||
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
|
||||
import { dependencyCover, HEALTH_STATE_FLAG } from "./dependencies.js";
|
||||
import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { hostHealthConditions, parseHostHealth, withoutIgnoredServices, type HostHealth } from "./hostHealth.js";
|
||||
import { hostHealthConditions, parseHostHealth, onlyWatchedServices, type HostHealth } from "./hostHealth.js";
|
||||
import { newListeningPorts, newPortMessage, parseBaseline, portKey, type NewPort } from "./newPorts.js";
|
||||
import { listPresets, serviceName, serviceNames } from "./portPresets.js";
|
||||
import { containerLabel, containerPortName, containerPortNames, purgeStaleContainerPorts, refreshContainerPorts } from "./containerPorts.js";
|
||||
@@ -558,7 +558,7 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
// a malformed report shouldn't stop every other server being checked
|
||||
}
|
||||
const newPorts = checks.newPortAlerts ? newListeningPorts(parseJson<AgentPort[] | null>(s.listeningPorts, null), parseBaseline(s.portBaseline), expectedByServer.get(s.id) ?? new Set()).map((p) => ({ ...p, service: containerLabel(containerPortName(containerNames, s.id, "tcp", p.port)) ?? serviceName(portNames, "tcp", p.port) })) : [];
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: withoutIgnoredServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.ignoreServices), thresholds: parseThresholds(s.thresholds), newPorts });
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: onlyWatchedServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.watchServices), thresholds: parseThresholds(s.thresholds), newPorts });
|
||||
}
|
||||
|
||||
// Their own rules: what the stored history says. A failure here costs only the rules.
|
||||
|
||||
@@ -8,8 +8,8 @@
|
||||
export interface FailedUnit {
|
||||
unit: string;
|
||||
description: string;
|
||||
/** Set where it is shown, for a unit this server has been told to ignore. It never counts as failed. */
|
||||
ignored?: boolean;
|
||||
/** Set where it is shown: whether this server is told to watch it. A unit that is not watched never counts as failed. */
|
||||
watched?: boolean;
|
||||
}
|
||||
export interface SmartDisk {
|
||||
device: string;
|
||||
@@ -86,8 +86,8 @@ export function nextHostHealth(previous: string | null, incoming: IncomingHostHe
|
||||
|
||||
const globToRegex = (pattern: string) => new RegExp(`^${pattern.trim().toLowerCase().replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*")}$`);
|
||||
|
||||
/** Whether a unit is one this server is told to ignore: a name, or a pattern with * ("gupdate*"); case doesn't matter and ".service" may be left off. */
|
||||
export function isServiceIgnored(unit: string, patterns: string[] | undefined): boolean {
|
||||
/** Whether a unit is one of those a server is told to watch: a name, or a pattern with * ("docker*"; "*" is every service); case doesn't matter and ".service" may be left off. */
|
||||
export function isServiceWatched(unit: string, patterns: string[] | undefined): boolean {
|
||||
if (!patterns || patterns.length === 0) return false;
|
||||
const u = unit.toLowerCase();
|
||||
const bare = u.replace(/\.service$/, "");
|
||||
@@ -97,16 +97,19 @@ export function isServiceIgnored(unit: string, patterns: string[] | undefined):
|
||||
});
|
||||
}
|
||||
|
||||
/** The health with the ignored services taken out: what alerts and figures are worked from. */
|
||||
export function withoutIgnoredServices(health: HostHealth | null, patterns: string[] | undefined): HostHealth | null {
|
||||
if (!health?.failedUnits || !patterns || patterns.length === 0) return health;
|
||||
return { ...health, failedUnits: health.failedUnits.filter((u) => !isServiceIgnored(u.unit, patterns)) };
|
||||
/**
|
||||
* The health with only the watched services kept: what alerts and figures are worked from. Services are opt-in — with nothing watched, no service counts as failed, however many the agent reports.
|
||||
* A report with no part about services at all (an agent that could not collect it) stays as it is.
|
||||
*/
|
||||
export function onlyWatchedServices(health: HostHealth | null, patterns: string[] | undefined): HostHealth | null {
|
||||
if (!health?.failedUnits) return health;
|
||||
return { ...health, failedUnits: health.failedUnits.filter((u) => isServiceWatched(u.unit, patterns)) };
|
||||
}
|
||||
|
||||
/** The health with the ignored services marked, not removed: what the server's page shows, so they can be seen and taken off the list. */
|
||||
export function markIgnoredServices(health: HostHealth | null, patterns: string[] | undefined): HostHealth | null {
|
||||
if (!health?.failedUnits || !patterns || patterns.length === 0) return health;
|
||||
return { ...health, failedUnits: health.failedUnits.map((u) => (isServiceIgnored(u.unit, patterns) ? { ...u, ignored: true } : u)) };
|
||||
/** The health with each failed service marked as watched or not, none removed: what the server's page shows, so a failed service can be seen and picked to be watched. */
|
||||
export function markWatchedServices(health: HostHealth | null, patterns: string[] | undefined): HostHealth | null {
|
||||
if (!health?.failedUnits) return health;
|
||||
return { ...health, failedUnits: health.failedUnits.map((u) => ({ ...u, watched: isServiceWatched(u.unit, patterns) })) };
|
||||
}
|
||||
|
||||
/** A pool in any state but ONLINE has lost something (DEGRADED has lost redundancy, the rest are worse). */
|
||||
|
||||
@@ -36,7 +36,7 @@ import { getAlerts } from "./alerts.js";
|
||||
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
|
||||
import { getSourceStatus } from "./integrationHealthMonitor.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { parseHostHealth, poolIsFine, withoutIgnoredServices } from "./hostHealth.js";
|
||||
import { parseHostHealth, poolIsFine, onlyWatchedServices } from "./hostHealth.js";
|
||||
import { parseThresholds as parseServerThresholds } from "./serverThresholds.js";
|
||||
import { parseThresholds } from "./serverThresholds.js";
|
||||
import { getCachedBackupCoverage } from "./backupCoverage.js";
|
||||
@@ -534,7 +534,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
if (updates.security !== null) out.add("homelab_server_pending_security_updates", "gauge", "Of those, the ones that are security fixes.", updates.security, { server });
|
||||
out.add("homelab_server_reboot_required", "gauge", "1 if a restart is due.", updates.rebootRequired ? 1 : 0, { server });
|
||||
}
|
||||
const health = withoutIgnoredServices(parseHostHealth(s.hostHealth), parseServerThresholds(s.thresholds)?.ignoreServices);
|
||||
const health = onlyWatchedServices(parseHostHealth(s.hostHealth), parseServerThresholds(s.thresholds)?.watchServices);
|
||||
if (health?.failedUnits) out.add("homelab_server_failed_units", "gauge", "Services (systemd units) that have failed on the server.", health.failedUnits.length, { server });
|
||||
for (const d of health?.smart ?? []) {
|
||||
if (d.healthy !== null) out.add("homelab_server_disk_healthy", "gauge", "1 if the disk's own SMART self-assessment passes, 0 if it fails.", d.healthy ? 1 : 0, { server, device: d.device });
|
||||
|
||||
@@ -10,7 +10,7 @@ import { attentionCounts, loadAcks } from "./alertAcks.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { loadStats } from "./endpointChecks.js";
|
||||
import { describeTarget, listActiveWindows, listUpcomingWindows } from "./maintenance.js";
|
||||
import { parseHostHealth } from "./hostHealth.js";
|
||||
import { onlyWatchedServices, parseHostHealth } from "./hostHealth.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { parseTags } from "./serverTags.js";
|
||||
import { parseThresholds } from "./serverThresholds.js";
|
||||
@@ -38,7 +38,7 @@ export function serverV1(row: ServerRow, offlineMinutes: number, now: Date, osWa
|
||||
const status = serverStatus(row, offlineMinutes, now);
|
||||
const disks = json<{ mount: string; sizeBytes: number; usedBytes: number }[]>(row.disks, []);
|
||||
const updates = parsePendingUpdates(row.pendingUpdates);
|
||||
const health = parseHostHealth(row.hostHealth);
|
||||
const health = onlyWatchedServices(parseHostHealth(row.hostHealth), parseThresholds(row.thresholds)?.watchServices);
|
||||
return {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
|
||||
@@ -21,8 +21,8 @@ export interface ServerThresholds {
|
||||
/** 0 turns the stale-backup alert off for this server's guest. */
|
||||
backupStaleHours?: number;
|
||||
mounts?: Record<string, MountThreshold>;
|
||||
/** Services (units) never to count as failed on this server: names or patterns like "gupdate*" (case does not matter, ".service" is optional). */
|
||||
ignoreServices?: string[];
|
||||
/** The services (units) to watch on this server: names or patterns like "docker*" (case does not matter, ".service" is optional; "*" is all of them). Only a watched service that has failed raises an alert. */
|
||||
watchServices?: string[];
|
||||
}
|
||||
|
||||
/** The global values these override. */
|
||||
@@ -40,7 +40,7 @@ export const LIMITS = {
|
||||
backupStaleHours: { min: 0, max: 2160 },
|
||||
} as const;
|
||||
export const MAX_MOUNT_RULES = 50;
|
||||
export const MAX_IGNORED_SERVICES = 30;
|
||||
export const MAX_WATCHED_SERVICES = 50;
|
||||
const MAX_SERVICE_PATTERN = 100;
|
||||
|
||||
const inRange = (v: unknown, key: keyof typeof LIMITS): v is number => typeof v === "number" && Number.isInteger(v) && v >= LIMITS[key].min && v <= LIMITS[key].max;
|
||||
@@ -65,10 +65,10 @@ export function parseThresholds(stored: string | null | undefined): ServerThresh
|
||||
}
|
||||
if (Object.keys(mounts).length > 0) out.mounts = mounts;
|
||||
}
|
||||
if (Array.isArray(v.ignoreServices)) {
|
||||
const strings: string[] = (v.ignoreServices as unknown[]).filter((p): p is string => typeof p === "string");
|
||||
const list = [...new Set(strings.map((p) => p.trim()).filter((p) => p.length > 0 && p.length <= MAX_SERVICE_PATTERN))].slice(0, MAX_IGNORED_SERVICES);
|
||||
if (list.length > 0) out.ignoreServices = list;
|
||||
if (Array.isArray(v.watchServices)) {
|
||||
const strings: string[] = (v.watchServices as unknown[]).filter((p): p is string => typeof p === "string");
|
||||
const list = [...new Set(strings.map((p) => p.trim()).filter((p) => p.length > 0 && p.length <= MAX_SERVICE_PATTERN))].slice(0, MAX_WATCHED_SERVICES);
|
||||
if (list.length > 0) out.watchServices = list;
|
||||
}
|
||||
return Object.keys(out).length > 0 ? out : null;
|
||||
} catch {
|
||||
@@ -112,19 +112,19 @@ export function checkThresholds(input: unknown): { ok: true; value: ServerThresh
|
||||
}
|
||||
if (Object.keys(mounts).length > 0) out.mounts = mounts;
|
||||
}
|
||||
if (v.ignoreServices !== undefined && v.ignoreServices !== null) {
|
||||
if (!Array.isArray(v.ignoreServices)) return { ok: false, message: "The services to ignore should be a list." };
|
||||
if (v.watchServices !== undefined && v.watchServices !== null) {
|
||||
if (!Array.isArray(v.watchServices)) return { ok: false, message: "The services to watch should be a list." };
|
||||
const list: string[] = [];
|
||||
for (const raw of v.ignoreServices) {
|
||||
if (typeof raw !== "string") return { ok: false, message: "Each service to ignore should be a name or a pattern." };
|
||||
for (const raw of v.watchServices) {
|
||||
if (typeof raw !== "string") return { ok: false, message: "Each service to watch should be a name or a pattern." };
|
||||
const p = raw.trim();
|
||||
if (p.length === 0) continue;
|
||||
// eslint-disable-next-line no-control-regex
|
||||
if (p.length > MAX_SERVICE_PATTERN || /[\u0000-\u001f]/.test(p)) return { ok: false, message: `"${p.slice(0, 30)}…" isn't a service name or pattern (up to ${MAX_SERVICE_PATTERN} characters, no line breaks).` };
|
||||
if (!list.includes(p)) list.push(p);
|
||||
}
|
||||
if (list.length > MAX_IGNORED_SERVICES) return { ok: false, message: `At most ${MAX_IGNORED_SERVICES} services can be ignored on one server.` };
|
||||
if (list.length > 0) out.ignoreServices = list;
|
||||
if (list.length > MAX_WATCHED_SERVICES) return { ok: false, message: `At most ${MAX_WATCHED_SERVICES} services can be watched on one server.` };
|
||||
if (list.length > 0) out.watchServices = list;
|
||||
}
|
||||
return { ok: true, value: Object.keys(out).length > 0 ? out : null };
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ const { db } = await import("../src/db/client.js");
|
||||
const { users, servers } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { describeDiagnostics, parseDiagnostics } = await import("../src/services/agentDiagnostics.js");
|
||||
const { isServiceIgnored, withoutIgnoredServices, markIgnoredServices } = await import("../src/services/hostHealth.js");
|
||||
const { isServiceWatched, onlyWatchedServices, markWatchedServices } = await import("../src/services/hostHealth.js");
|
||||
const { checkThresholds, parseThresholds } = await import("../src/services/serverThresholds.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
@@ -89,36 +89,44 @@ describe("saying what to do about what couldn't be collected", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("services a server is told to ignore", () => {
|
||||
describe("services a server is told to watch", () => {
|
||||
test("names and patterns, in any case, with or without .service", () => {
|
||||
assert.equal(isServiceIgnored("gupdate.service", ["gupdate"]), true);
|
||||
assert.equal(isServiceIgnored("GUpdate", ["gupdate*"]), true);
|
||||
assert.equal(isServiceIgnored("snap.lxd.activate.service", ["snap.*"]), true);
|
||||
assert.equal(isServiceIgnored("nginx.service", ["gupdate*", "snap.*"]), false);
|
||||
assert.equal(isServiceIgnored("nginx.service", ["nginx.service"]), true);
|
||||
assert.equal(isServiceIgnored("mynginx.service", ["nginx"]), false, "a name is the whole name");
|
||||
assert.equal(isServiceIgnored("a+b.service", ["a+b"]), true, "characters that mean something to a regex are only characters here");
|
||||
assert.equal(isServiceIgnored("anything", []), false);
|
||||
assert.equal(isServiceIgnored("anything", undefined), false);
|
||||
assert.equal(isServiceWatched("gupdate.service", ["gupdate"]), true);
|
||||
assert.equal(isServiceWatched("GUpdate", ["gupdate*"]), true);
|
||||
assert.equal(isServiceWatched("snap.lxd.activate.service", ["snap.*"]), true);
|
||||
assert.equal(isServiceWatched("nginx.service", ["gupdate*", "snap.*"]), false);
|
||||
assert.equal(isServiceWatched("nginx.service", ["nginx.service"]), true);
|
||||
assert.equal(isServiceWatched("mynginx.service", ["nginx"]), false, "a name is the whole name");
|
||||
assert.equal(isServiceWatched("a+b.service", ["a+b"]), true, "characters that mean something to a regex are only characters here");
|
||||
assert.equal(isServiceWatched("anything", []), false);
|
||||
assert.equal(isServiceWatched("anything", undefined), false);
|
||||
});
|
||||
|
||||
test("left out for what is judged, marked for what is shown", () => {
|
||||
test("* is every service", () => {
|
||||
assert.equal(isServiceWatched("anything.service", ["*"]), true);
|
||||
});
|
||||
|
||||
test("only the watched are kept for what is judged, all are shown marked", () => {
|
||||
const health = { failedUnits: [{ unit: "nginx.service", description: "" }, { unit: "gupdate", description: "Google Update" }], reportedAt: "t" };
|
||||
assert.deepEqual(withoutIgnoredServices(health, ["gupdate*"])!.failedUnits!.map((u) => u.unit), ["nginx.service"]);
|
||||
assert.deepEqual(markIgnoredServices(health, ["gupdate*"])!.failedUnits!.map((u) => [u.unit, u.ignored ?? false]), [["nginx.service", false], ["gupdate", true]]);
|
||||
assert.equal(withoutIgnoredServices(null, ["x"]), null);
|
||||
assert.deepEqual(withoutIgnoredServices(health, []), health);
|
||||
assert.deepEqual(onlyWatchedServices(health, ["nginx"])!.failedUnits!.map((u) => u.unit), ["nginx.service"]);
|
||||
assert.deepEqual(markWatchedServices(health, ["nginx"])!.failedUnits!.map((u) => [u.unit, u.watched]), [["nginx.service", true], ["gupdate", false]]);
|
||||
assert.equal(onlyWatchedServices(null, ["x"]), null);
|
||||
assert.deepEqual(onlyWatchedServices(health, [])!.failedUnits, [], "nothing is watched until it is said: no service counts as failed");
|
||||
assert.deepEqual(onlyWatchedServices(health, undefined)!.failedUnits, []);
|
||||
assert.deepEqual(markWatchedServices(health, [])!.failedUnits!.map((u) => u.watched), [false, false]);
|
||||
const silent = { reportedAt: "t" };
|
||||
assert.deepEqual(onlyWatchedServices(silent, []), silent, "a report with nothing about services is left as it is");
|
||||
});
|
||||
|
||||
test("the list is part of a server's own thresholds: kept tidy, and a wrong one is refused", () => {
|
||||
const ok = checkThresholds({ ignoreServices: [" gupdate* ", "gupdate*", "", "snap.*"] });
|
||||
assert.deepEqual(ok.ok && ok.value, { ignoreServices: ["gupdate*", "snap.*"] });
|
||||
assert.equal(checkThresholds({ ignoreServices: "gupdate" }).ok, false);
|
||||
assert.equal(checkThresholds({ ignoreServices: [3] }).ok, false);
|
||||
assert.equal(checkThresholds({ ignoreServices: ["x".repeat(101)] }).ok, false);
|
||||
assert.equal(checkThresholds({ ignoreServices: Array.from({ length: 31 }, (_, i) => `s${i}`) }).ok, false);
|
||||
assert.equal(checkThresholds({ ignoreServices: [] }).ok && (checkThresholds({ ignoreServices: [] }) as { value: unknown }).value, null, "an empty list is no thresholds at all");
|
||||
assert.deepEqual(parseThresholds(JSON.stringify({ ignoreServices: ["a", 3, "", "a"] })), { ignoreServices: ["a"] });
|
||||
const ok = checkThresholds({ watchServices: [" gupdate* ", "gupdate*", "", "snap.*"] });
|
||||
assert.deepEqual(ok.ok && ok.value, { watchServices: ["gupdate*", "snap.*"] });
|
||||
assert.equal(checkThresholds({ watchServices: "gupdate" }).ok, false);
|
||||
assert.equal(checkThresholds({ watchServices: [3] }).ok, false);
|
||||
assert.equal(checkThresholds({ watchServices: ["x".repeat(101)] }).ok, false);
|
||||
assert.equal(checkThresholds({ watchServices: Array.from({ length: 51 }, (_, i) => `s${i}`) }).ok, false);
|
||||
assert.equal(checkThresholds({ watchServices: [] }).ok && (checkThresholds({ watchServices: [] }) as { value: unknown }).value, null, "an empty list is no thresholds at all");
|
||||
assert.deepEqual(parseThresholds(JSON.stringify({ watchServices: ["a", 3, "", "a"] })), { watchServices: ["a"] });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -179,28 +187,29 @@ describe("through a report, the page, the alerts and the figures", () => {
|
||||
assert.equal(row.agentDiagnostics, undefined, "never as raw stored text");
|
||||
});
|
||||
|
||||
test("a service the server is told to ignore stops being an alert and a figure, still shows on the page marked, and can be un-ignored", async () => {
|
||||
test("nothing is an alert until it is watched; a watched service that has failed is, and a failed one that is not watched still shows on the page", async () => {
|
||||
assert.equal((await report({ health: { failed_units: [{ unit: "nginx.service", description: "web" }, { unit: "gupdate.service", description: "Google Update" }] } })).status, 202);
|
||||
invalidateAlertsCache();
|
||||
const failing = async () => (await getAlerts(true)).alerts.find((a) => a.id.startsWith("services:failed-units") && a.id.includes(`:server:${id}`));
|
||||
assert.match((await failing())!.message, /2 services have failed/);
|
||||
assert.equal(await failing(), undefined, "nothing is watched, so nothing is an alert");
|
||||
assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\{server="elsa"[^}]*\} 0`));
|
||||
assert.deepEqual((await detail()).server.hostHealth.failedUnits.map((u: { unit: string; watched?: boolean }) => [u.unit, u.watched]), [["nginx.service", false], ["gupdate.service", false]], "still shown, marked not watched");
|
||||
|
||||
assert.equal((await patch({ thresholds: { ignoreServices: ["gupdate*"] } })).status, 200);
|
||||
assert.equal((await patch({ thresholds: { watchServices: ["nginx"] } })).status, 200);
|
||||
assert.match((await failing())!.message, /1 service has failed — nginx\.service/, "the alert cache was dropped when the list changed");
|
||||
assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\\{server="elsa"[^}]*\\} 1`));
|
||||
|
||||
assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\{server="elsa"[^}]*\} 1`));
|
||||
const units = (await detail()).server.hostHealth.failedUnits;
|
||||
assert.deepEqual(units.map((u: { unit: string; ignored?: boolean }) => [u.unit, u.ignored ?? false]), [["nginx.service", false], ["gupdate.service", true]]);
|
||||
assert.deepEqual(units.map((u: { unit: string; watched?: boolean }) => [u.unit, u.watched]), [["nginx.service", true], ["gupdate.service", false]]);
|
||||
|
||||
assert.equal((await patch({ thresholds: { ignoreServices: ["gupdate*", "nginx"] } })).status, 200);
|
||||
assert.equal(await failing(), undefined, "with every failed service ignored there is nothing to alert about");
|
||||
assert.equal((await patch({ thresholds: { watchServices: ["*"] } })).status, 200);
|
||||
assert.match((await failing())!.message, /2 services have failed/, "* watches them all, as it always was");
|
||||
|
||||
assert.equal((await patch({ thresholds: null })).status, 200);
|
||||
assert.match((await failing())!.message, /2 services have failed/);
|
||||
assert.equal(await failing(), undefined, "taking the list away watches nothing again");
|
||||
});
|
||||
|
||||
test("a list that isn't one is refused with a message", async () => {
|
||||
const res = await patch({ thresholds: { ignoreServices: "gupdate" } });
|
||||
const res = await patch({ thresholds: { watchServices: "gupdate" } });
|
||||
assert.equal(res.status, 400);
|
||||
assert.match((await res.json()).message, /should be a list/);
|
||||
});
|
||||
|
||||
@@ -273,7 +273,7 @@ describe("the noise report", () => {
|
||||
assert.match(hintFor("monitoring", "monitoring:new-port:server:1:tcp:8080"), /Ports card/);
|
||||
assert.match(hintFor("automation", "automation:heartbeat:1"), /grace/);
|
||||
assert.match(hintFor("monitoring", "monitoring:endpoint:2"), /failures in a row/);
|
||||
assert.match(hintFor("services", "services:failed-units:server:1"), /Services to ignore/);
|
||||
assert.match(hintFor("services", "services:failed-units:server:1"), /Services to watch/);
|
||||
assert.match(hintFor("tickets", "tickets:x"), /Acknowledge it/);
|
||||
});
|
||||
|
||||
|
||||
@@ -109,7 +109,7 @@ describe("through the report, the alerts, the figures and the page", () => {
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olle", role: "operator" },
|
||||
]);
|
||||
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t" }).returning();
|
||||
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", thresholds: JSON.stringify({ watchServices: ["*"] }) }).returning();
|
||||
id = s.id;
|
||||
const a = express();
|
||||
a.use("/api/agent/report", agentReportRouter);
|
||||
|
||||
Reference in new issue
Block a user