Services: opt-in monitoring with a per-server watch list
A failed service now raises an alert (and counts in /metrics and the public API) only when the server is told to watch it, by name or pattern, or * for all. Failed services that are not watched still show on the Host health card with a Watch button. Replaces the ignore list. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6e39430a1f
commit
06ac120a1d
19 files changed
+143
-121
No files matched your search
@@ -22,7 +22,7 @@ const { db } = await import("../src/db/client.js");
|
||||
const { users, servers } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { describeDiagnostics, parseDiagnostics } = await import("../src/services/agentDiagnostics.js");
|
||||
const { isServiceIgnored, withoutIgnoredServices, markIgnoredServices } = await import("../src/services/hostHealth.js");
|
||||
const { isServiceWatched, onlyWatchedServices, markWatchedServices } = await import("../src/services/hostHealth.js");
|
||||
const { checkThresholds, parseThresholds } = await import("../src/services/serverThresholds.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
@@ -89,36 +89,44 @@ describe("saying what to do about what couldn't be collected", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("services a server is told to ignore", () => {
|
||||
describe("services a server is told to watch", () => {
|
||||
test("names and patterns, in any case, with or without .service", () => {
|
||||
assert.equal(isServiceIgnored("gupdate.service", ["gupdate"]), true);
|
||||
assert.equal(isServiceIgnored("GUpdate", ["gupdate*"]), true);
|
||||
assert.equal(isServiceIgnored("snap.lxd.activate.service", ["snap.*"]), true);
|
||||
assert.equal(isServiceIgnored("nginx.service", ["gupdate*", "snap.*"]), false);
|
||||
assert.equal(isServiceIgnored("nginx.service", ["nginx.service"]), true);
|
||||
assert.equal(isServiceIgnored("mynginx.service", ["nginx"]), false, "a name is the whole name");
|
||||
assert.equal(isServiceIgnored("a+b.service", ["a+b"]), true, "characters that mean something to a regex are only characters here");
|
||||
assert.equal(isServiceIgnored("anything", []), false);
|
||||
assert.equal(isServiceIgnored("anything", undefined), false);
|
||||
assert.equal(isServiceWatched("gupdate.service", ["gupdate"]), true);
|
||||
assert.equal(isServiceWatched("GUpdate", ["gupdate*"]), true);
|
||||
assert.equal(isServiceWatched("snap.lxd.activate.service", ["snap.*"]), true);
|
||||
assert.equal(isServiceWatched("nginx.service", ["gupdate*", "snap.*"]), false);
|
||||
assert.equal(isServiceWatched("nginx.service", ["nginx.service"]), true);
|
||||
assert.equal(isServiceWatched("mynginx.service", ["nginx"]), false, "a name is the whole name");
|
||||
assert.equal(isServiceWatched("a+b.service", ["a+b"]), true, "characters that mean something to a regex are only characters here");
|
||||
assert.equal(isServiceWatched("anything", []), false);
|
||||
assert.equal(isServiceWatched("anything", undefined), false);
|
||||
});
|
||||
|
||||
test("left out for what is judged, marked for what is shown", () => {
|
||||
test("* is every service", () => {
|
||||
assert.equal(isServiceWatched("anything.service", ["*"]), true);
|
||||
});
|
||||
|
||||
test("only the watched are kept for what is judged, all are shown marked", () => {
|
||||
const health = { failedUnits: [{ unit: "nginx.service", description: "" }, { unit: "gupdate", description: "Google Update" }], reportedAt: "t" };
|
||||
assert.deepEqual(withoutIgnoredServices(health, ["gupdate*"])!.failedUnits!.map((u) => u.unit), ["nginx.service"]);
|
||||
assert.deepEqual(markIgnoredServices(health, ["gupdate*"])!.failedUnits!.map((u) => [u.unit, u.ignored ?? false]), [["nginx.service", false], ["gupdate", true]]);
|
||||
assert.equal(withoutIgnoredServices(null, ["x"]), null);
|
||||
assert.deepEqual(withoutIgnoredServices(health, []), health);
|
||||
assert.deepEqual(onlyWatchedServices(health, ["nginx"])!.failedUnits!.map((u) => u.unit), ["nginx.service"]);
|
||||
assert.deepEqual(markWatchedServices(health, ["nginx"])!.failedUnits!.map((u) => [u.unit, u.watched]), [["nginx.service", true], ["gupdate", false]]);
|
||||
assert.equal(onlyWatchedServices(null, ["x"]), null);
|
||||
assert.deepEqual(onlyWatchedServices(health, [])!.failedUnits, [], "nothing is watched until it is said: no service counts as failed");
|
||||
assert.deepEqual(onlyWatchedServices(health, undefined)!.failedUnits, []);
|
||||
assert.deepEqual(markWatchedServices(health, [])!.failedUnits!.map((u) => u.watched), [false, false]);
|
||||
const silent = { reportedAt: "t" };
|
||||
assert.deepEqual(onlyWatchedServices(silent, []), silent, "a report with nothing about services is left as it is");
|
||||
});
|
||||
|
||||
test("the list is part of a server's own thresholds: kept tidy, and a wrong one is refused", () => {
|
||||
const ok = checkThresholds({ ignoreServices: [" gupdate* ", "gupdate*", "", "snap.*"] });
|
||||
assert.deepEqual(ok.ok && ok.value, { ignoreServices: ["gupdate*", "snap.*"] });
|
||||
assert.equal(checkThresholds({ ignoreServices: "gupdate" }).ok, false);
|
||||
assert.equal(checkThresholds({ ignoreServices: [3] }).ok, false);
|
||||
assert.equal(checkThresholds({ ignoreServices: ["x".repeat(101)] }).ok, false);
|
||||
assert.equal(checkThresholds({ ignoreServices: Array.from({ length: 31 }, (_, i) => `s${i}`) }).ok, false);
|
||||
assert.equal(checkThresholds({ ignoreServices: [] }).ok && (checkThresholds({ ignoreServices: [] }) as { value: unknown }).value, null, "an empty list is no thresholds at all");
|
||||
assert.deepEqual(parseThresholds(JSON.stringify({ ignoreServices: ["a", 3, "", "a"] })), { ignoreServices: ["a"] });
|
||||
const ok = checkThresholds({ watchServices: [" gupdate* ", "gupdate*", "", "snap.*"] });
|
||||
assert.deepEqual(ok.ok && ok.value, { watchServices: ["gupdate*", "snap.*"] });
|
||||
assert.equal(checkThresholds({ watchServices: "gupdate" }).ok, false);
|
||||
assert.equal(checkThresholds({ watchServices: [3] }).ok, false);
|
||||
assert.equal(checkThresholds({ watchServices: ["x".repeat(101)] }).ok, false);
|
||||
assert.equal(checkThresholds({ watchServices: Array.from({ length: 51 }, (_, i) => `s${i}`) }).ok, false);
|
||||
assert.equal(checkThresholds({ watchServices: [] }).ok && (checkThresholds({ watchServices: [] }) as { value: unknown }).value, null, "an empty list is no thresholds at all");
|
||||
assert.deepEqual(parseThresholds(JSON.stringify({ watchServices: ["a", 3, "", "a"] })), { watchServices: ["a"] });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -179,28 +187,29 @@ describe("through a report, the page, the alerts and the figures", () => {
|
||||
assert.equal(row.agentDiagnostics, undefined, "never as raw stored text");
|
||||
});
|
||||
|
||||
test("a service the server is told to ignore stops being an alert and a figure, still shows on the page marked, and can be un-ignored", async () => {
|
||||
test("nothing is an alert until it is watched; a watched service that has failed is, and a failed one that is not watched still shows on the page", async () => {
|
||||
assert.equal((await report({ health: { failed_units: [{ unit: "nginx.service", description: "web" }, { unit: "gupdate.service", description: "Google Update" }] } })).status, 202);
|
||||
invalidateAlertsCache();
|
||||
const failing = async () => (await getAlerts(true)).alerts.find((a) => a.id.startsWith("services:failed-units") && a.id.includes(`:server:${id}`));
|
||||
assert.match((await failing())!.message, /2 services have failed/);
|
||||
assert.equal(await failing(), undefined, "nothing is watched, so nothing is an alert");
|
||||
assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\{server="elsa"[^}]*\} 0`));
|
||||
assert.deepEqual((await detail()).server.hostHealth.failedUnits.map((u: { unit: string; watched?: boolean }) => [u.unit, u.watched]), [["nginx.service", false], ["gupdate.service", false]], "still shown, marked not watched");
|
||||
|
||||
assert.equal((await patch({ thresholds: { ignoreServices: ["gupdate*"] } })).status, 200);
|
||||
assert.equal((await patch({ thresholds: { watchServices: ["nginx"] } })).status, 200);
|
||||
assert.match((await failing())!.message, /1 service has failed — nginx\.service/, "the alert cache was dropped when the list changed");
|
||||
assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\\{server="elsa"[^}]*\\} 1`));
|
||||
|
||||
assert.match(await buildMetrics(), new RegExp(`homelab_server_failed_units\{server="elsa"[^}]*\} 1`));
|
||||
const units = (await detail()).server.hostHealth.failedUnits;
|
||||
assert.deepEqual(units.map((u: { unit: string; ignored?: boolean }) => [u.unit, u.ignored ?? false]), [["nginx.service", false], ["gupdate.service", true]]);
|
||||
assert.deepEqual(units.map((u: { unit: string; watched?: boolean }) => [u.unit, u.watched]), [["nginx.service", true], ["gupdate.service", false]]);
|
||||
|
||||
assert.equal((await patch({ thresholds: { ignoreServices: ["gupdate*", "nginx"] } })).status, 200);
|
||||
assert.equal(await failing(), undefined, "with every failed service ignored there is nothing to alert about");
|
||||
assert.equal((await patch({ thresholds: { watchServices: ["*"] } })).status, 200);
|
||||
assert.match((await failing())!.message, /2 services have failed/, "* watches them all, as it always was");
|
||||
|
||||
assert.equal((await patch({ thresholds: null })).status, 200);
|
||||
assert.match((await failing())!.message, /2 services have failed/);
|
||||
assert.equal(await failing(), undefined, "taking the list away watches nothing again");
|
||||
});
|
||||
|
||||
test("a list that isn't one is refused with a message", async () => {
|
||||
const res = await patch({ thresholds: { ignoreServices: "gupdate" } });
|
||||
const res = await patch({ thresholds: { watchServices: "gupdate" } });
|
||||
assert.equal(res.status, 400);
|
||||
assert.match((await res.json()).message, /should be a list/);
|
||||
});
|
||||
|
||||
@@ -273,7 +273,7 @@ describe("the noise report", () => {
|
||||
assert.match(hintFor("monitoring", "monitoring:new-port:server:1:tcp:8080"), /Ports card/);
|
||||
assert.match(hintFor("automation", "automation:heartbeat:1"), /grace/);
|
||||
assert.match(hintFor("monitoring", "monitoring:endpoint:2"), /failures in a row/);
|
||||
assert.match(hintFor("services", "services:failed-units:server:1"), /Services to ignore/);
|
||||
assert.match(hintFor("services", "services:failed-units:server:1"), /Services to watch/);
|
||||
assert.match(hintFor("tickets", "tickets:x"), /Acknowledge it/);
|
||||
});
|
||||
|
||||
|
||||
@@ -109,7 +109,7 @@ describe("through the report, the alerts, the figures and the page", () => {
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olle", role: "operator" },
|
||||
]);
|
||||
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t" }).returning();
|
||||
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", thresholds: JSON.stringify({ watchServices: ["*"] }) }).returning();
|
||||
id = s.id;
|
||||
const a = express();
|
||||
a.use("/api/agent/report", agentReportRouter);
|
||||
|
||||
Reference in new issue
Block a user