Show cluster health in the Technitium DNS integration
Reads /api/admin/cluster/state (Technitium 14+): a Cluster table on the page, alerts for a node unreachable for ten minutes (primary critical) and for a secondary that has stopped taking the primary's configuration, and three /metrics series. A server in no cluster shows nothing; a token that may not view administration is told so and the rest of the read is unaffected. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1ef686d1ba
commit
d78aba8bf9
15 files changed
+9423
-15
No files matched your search
@@ -160,6 +160,9 @@ describe("the adapter against a Technitium that answers", () => {
|
||||
let dhcpScopes: unknown[] = [{ name: "lan", enabled: true }];
|
||||
/** The port the fake settings say the DNS server is on; the one of the answering fake DNS server unless a test points it elsewhere. */
|
||||
let endpointPort: number | null = null;
|
||||
/** What the fake cluster call answers: nothing (a server with no such call), a refusal of the token's user, a failure, or a cluster. */
|
||||
let clusterMode: "none" | "denied" | "boom" | "cluster" = "none";
|
||||
const clusterBody = { clusterInitialized: true, dnsServerDomain: "dns2.example.com", version: "15.6", clusterDomain: "example.com", heartbeatRefreshIntervalSeconds: 30, configRefreshIntervalSeconds: 900, configLastSynced: "2026-09-26T12:30:16.1234567Z", clusterNodes: [{ id: 2, name: "dns2.example.com", url: "https://dns2.example.com:53443/", ipAddress: "10.0.0.3", type: "Secondary", state: "Self", version: "15.6" }, { id: 3, name: "dns3.example.com", type: "Secondary", state: "Unreachable", lastSeen: "0001-01-01T00:00:00" }, { id: 1, name: "dns1.example.com", url: "https://dns1.example.com:53443/", ipAddresses: ["10.0.0.2"], type: "Primary", state: "Connected", version: "15.6", lastSeen: "2026-09-26T12:30:16Z" }] };
|
||||
|
||||
before(async () => {
|
||||
dns = await fakeDns(() => ({}));
|
||||
@@ -186,6 +189,11 @@ describe("the adapter against a Technitium that answers", () => {
|
||||
return ok({ scopes: dhcpScopes });
|
||||
case "/api/dhcp/leases/list":
|
||||
return ok({ leases: [{}, {}] });
|
||||
case "/api/admin/cluster/state":
|
||||
if (clusterMode === "denied") return send(200, JSON.stringify({ status: "error", errorMessage: "Access was denied." }));
|
||||
if (clusterMode === "boom") return send(500, "{}");
|
||||
if (clusterMode === "cluster") return ok(clusterBody);
|
||||
return send(404, "{}");
|
||||
default:
|
||||
return send(404, "{}");
|
||||
}
|
||||
@@ -205,13 +213,14 @@ describe("the adapter against a Technitium that answers", () => {
|
||||
httpStatus = 200;
|
||||
rawBody = null;
|
||||
dhcpScopes = [{ name: "lan", enabled: true }];
|
||||
clusterMode = "none";
|
||||
};
|
||||
|
||||
test("a read puts the token in the Authorization header, asks the five things and one DNS question, and says what each answered", async () => {
|
||||
test("a read puts the token in the Authorization header, asks the six things and one DNS question, and says what each answered", async () => {
|
||||
reset();
|
||||
const r = await adapter().read();
|
||||
assert.deepEqual([...new Set(seen.map((s) => s.auth))], ["Bearer tok-1"]);
|
||||
assert.deepEqual(seen.map((s) => s.path).sort(), ["/api/dashboard/stats/get?type=LastDay", "/api/dhcp/leases/list", "/api/dhcp/scopes/list", "/api/settings/get", "/api/zones/list?pageNumber=1&zonesPerPage=1000"]);
|
||||
assert.deepEqual(seen.map((s) => s.path).sort(), ["/api/admin/cluster/state", "/api/dashboard/stats/get?type=LastDay", "/api/dhcp/leases/list", "/api/dhcp/scopes/list", "/api/settings/get", "/api/zones/list?pageNumber=1&zonesPerPage=1000"]);
|
||||
assert.deepEqual([r.version, r.dnsPort, r.blockingEnabled, r.settings.blockLists.count, r.settings.cacheMaximumEntries], ["15.5", dns.port, true, 1, 10000]);
|
||||
assert.deepEqual([r.stats?.queries, r.stats?.blockedPercent, r.zones?.issues, r.dhcp], [1000, 20, [{ name: "lab.lan", problem: "expired" }], { scopes: 1, enabledScopes: 1, leases: 2 }]);
|
||||
assert.deepEqual([r.probe.ok, r.probe.name, r.probe.host], [true, "example.com", "127.0.0.1"]);
|
||||
@@ -228,6 +237,30 @@ describe("the adapter against a Technitium that answers", () => {
|
||||
assert.equal(seen.some((s) => s.path === "/api/dhcp/leases/list"), false);
|
||||
});
|
||||
|
||||
test("the cluster: a server with no such call is in none, one in a cluster says its nodes, and neither costs the rest of the read", async () => {
|
||||
reset();
|
||||
const none = await adapter().read();
|
||||
assert.deepEqual(none.cluster, { info: null, note: null, denied: false });
|
||||
clusterMode = "cluster";
|
||||
const r = await adapter().read();
|
||||
assert.equal(r.cluster?.note, null);
|
||||
assert.deepEqual([r.cluster?.info?.domain, r.cluster?.info?.thisNode, r.cluster?.info?.nodes.map((n) => `${n.name}=${n.type}/${n.state}`)], ["example.com", "dns2.example.com", ["dns1.example.com=Primary/Connected", "dns2.example.com=Secondary/Self", "dns3.example.com=Secondary/Unreachable"]]);
|
||||
assert.equal(r.stats?.queries, 1000, "the rest of the read is as before");
|
||||
});
|
||||
|
||||
test("the cluster: a token that may not view it is told so, a read that failed this time is said, and the rest of the read still works", async () => {
|
||||
reset();
|
||||
clusterMode = "denied";
|
||||
const denied = await adapter().read();
|
||||
assert.equal(denied.cluster?.denied, true);
|
||||
assert.match(denied.cluster?.note ?? "", /not allowed to view the cluster/);
|
||||
clusterMode = "boom";
|
||||
const boom = await adapter().read();
|
||||
assert.equal(boom.cluster?.denied, false);
|
||||
assert.match(boom.cluster?.note ?? "", /could not be read this time: Technitium error: HTTP 500/);
|
||||
assert.equal(boom.version, "15.5");
|
||||
});
|
||||
|
||||
test("what cannot be read is left out and the rest is kept: statistics, zones and DHCP are each soft; the settings are not", async () => {
|
||||
reset();
|
||||
failing = new Set(["/api/dashboard/stats/get", "/api/zones/list", "/api/dhcp/scopes/list"]);
|
||||
@@ -429,6 +462,151 @@ describe("what Technitium has stored", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("the cluster: reading it, carrying it, judging it", () => {
|
||||
const MINUTE = MIN;
|
||||
const t = iso;
|
||||
const NOW = Date.parse("2026-11-01T12:00:00Z");
|
||||
const node = (over: Partial<import("../src/integrations/technitium/adapter.js").TechnitiumClusterNode> = {}): import("../src/integrations/technitium/adapter.js").TechnitiumClusterNode => ({ id: 1, name: "dns1", type: "Primary", state: "Connected", url: null, version: "15.6", lastSeen: null, ipAddresses: [], ...over });
|
||||
const info = (nodes: ReturnType<typeof node>[], over: Partial<import("../src/integrations/technitium/adapter.js").TechnitiumCluster> = {}): import("../src/integrations/technitium/adapter.js").TechnitiumCluster => ({ domain: "example.com", thisNode: "dns2", heartbeatSeconds: 30, configRefreshSeconds: 900, configLastSynced: null, nodes, ...over });
|
||||
const cluster = (i: ReturnType<typeof info> | null, over: Partial<import("../src/services/technitiumSync.js").StoredCluster> = {}): import("../src/services/technitiumSync.js").StoredCluster => ({ info: i, note: null, unreachableSince: {}, ...over });
|
||||
|
||||
describe("parsing what Technitium says", () => {
|
||||
test("a server that is not in a cluster, or says nonsense, has none", () => {
|
||||
assert.equal(tn.parseCluster({ clusterInitialized: false, dnsServerDomain: "x", version: "15.6" }), null);
|
||||
for (const junk of [null, undefined, "x", [], {}, { clusterInitialized: "yes" }]) assert.equal(tn.parseCluster(junk), null, JSON.stringify(junk));
|
||||
});
|
||||
test("the nodes come primary first and then by name; a node with no name is left out; an address may be one or many, and never twice", () => {
|
||||
const c = tn.parseCluster({ clusterInitialized: true, clusterNodes: [{ name: "b", type: "Secondary", state: "Connected", ipAddress: "10.0.0.2", ipAddresses: ["10.0.0.2", "10.0.0.9"] }, { type: "Secondary" }, { name: "a", type: "Secondary", state: "Self" }, { name: "z", type: "Primary", state: "Connected" }] })!;
|
||||
assert.deepEqual(c.nodes.map((n) => n.name), ["z", "a", "b"]);
|
||||
assert.deepEqual(c.nodes[2].ipAddresses, ["10.0.0.2", "10.0.0.9"]);
|
||||
});
|
||||
test("a moment is read as Technitium writes it, and the year 1 it writes for never is none", () => {
|
||||
const c = tn.parseCluster({ clusterInitialized: true, configLastSynced: "2026-09-26T12:30:16.1234567Z", clusterNodes: [{ name: "a", lastSeen: "0001-01-01T00:00:00" }, { name: "b", lastSeen: "2026-09-26T12:30:16Z" }] })!;
|
||||
assert.equal(c.configLastSynced, "2026-09-26T12:30:16.123Z");
|
||||
assert.deepEqual(c.nodes.map((n) => [n.name, n.lastSeen]), [["a", null], ["b", "2026-09-26T12:30:16.000Z"]]);
|
||||
});
|
||||
test("what a node leaves out is unknown, not an error", () => {
|
||||
const c = tn.parseCluster({ clusterInitialized: true, clusterNodes: [{ name: "a" }] })!;
|
||||
assert.deepEqual([c.nodes[0].type, c.nodes[0].state, c.nodes[0].id, c.nodes[0].version, c.domain, c.configLastSynced], ["Unknown", "Unknown", null, null, null, null]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("carrying it between reads", () => {
|
||||
const read = (i: ReturnType<typeof info> | null, note: string | null = null, denied = false) => ({ info: i, note, denied });
|
||||
const down = info([node({ id: 3, name: "dns3", type: "Secondary", state: "Unreachable" })]);
|
||||
test("a node first seen unreachable starts its clock now, keeps it while it stays so, and loses it when it comes back", () => {
|
||||
const first = ts.carryCluster(read(down), null, new Date(NOW))!;
|
||||
assert.deepEqual(first.unreachableSince, { "3": t(NOW) });
|
||||
const later = ts.carryCluster(read(down), first, new Date(NOW + 5 * MINUTE))!;
|
||||
assert.deepEqual(later.unreachableSince, { "3": t(NOW) });
|
||||
const back = ts.carryCluster(read(info([node({ id: 3, name: "dns3", type: "Secondary" })])), later, new Date(NOW + 6 * MINUTE))!;
|
||||
assert.deepEqual(back.unreachableSince, {});
|
||||
});
|
||||
test("a node with no id is known by its name", () => {
|
||||
const c = ts.carryCluster(read(info([node({ id: null, name: "dnsX", state: "Unreachable" })])), null, new Date(NOW))!;
|
||||
assert.deepEqual(Object.keys(c.unreachableSince), ["dnsX"]);
|
||||
});
|
||||
test("a read that failed this time keeps what was known and the clocks, and says why; a refusal does not keep it", () => {
|
||||
const before = ts.carryCluster(read(down), null, new Date(NOW))!;
|
||||
const failed = ts.carryCluster(read(null, "boom"), before, new Date(NOW + 4 * MINUTE))!;
|
||||
assert.deepEqual([failed.info?.nodes.length, failed.note, failed.unreachableSince], [1, "boom", { "3": t(NOW) }]);
|
||||
const refused = ts.carryCluster(read(null, "not allowed", true), before, new Date(NOW + 4 * MINUTE))!;
|
||||
assert.deepEqual([refused.info, refused.note], [null, "not allowed"]);
|
||||
});
|
||||
test("a read made before the cluster was asked about leaves what is stored as it is", () => {
|
||||
const before = cluster(down);
|
||||
assert.equal(ts.carryCluster(undefined, before, new Date(NOW)), before);
|
||||
assert.equal(ts.carryCluster(undefined, null, new Date(NOW)), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is wrong", () => {
|
||||
const primaryDown = (extra: Partial<ReturnType<typeof node>> = {}) => node({ id: 1, name: "dns1", type: "Primary", state: "Unreachable", ...extra });
|
||||
const secondaryDown = (extra: Partial<ReturnType<typeof node>> = {}) => node({ id: 3, name: "dns3", type: "Secondary", state: "Unreachable", ...extra });
|
||||
const self = (extra: Partial<ReturnType<typeof node>> = {}) => node({ id: 2, name: "dns2", type: "Secondary", state: "Self", ...extra });
|
||||
const problems = (c: ReturnType<typeof cluster> | null, now = NOW) => ts.clusterProblems(7, "Home DNS", c, now);
|
||||
|
||||
test("nothing in a cluster that is well, in none, never asked about, or that could not be read", () => {
|
||||
assert.deepEqual(problems(cluster(info([node(), self()]))), []);
|
||||
assert.deepEqual(problems(null), []);
|
||||
assert.deepEqual(problems(cluster(null)), []);
|
||||
assert.deepEqual(problems(cluster(info([secondaryDown()]), { note: "boom", unreachableSince: { "3": t(NOW - HOUR) } })), []);
|
||||
});
|
||||
test("a node unreachable for ten minutes by this app's clock is an alert; a blip is not", () => {
|
||||
assert.deepEqual(problems(cluster(info([secondaryDown()]), { unreachableSince: { "3": t(NOW - 9 * MINUTE) } })), []);
|
||||
const p = problems(cluster(info([secondaryDown()]), { unreachableSince: { "3": t(NOW - 12 * MINUTE) } }));
|
||||
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["technitium-node:7:3", "warning"]]);
|
||||
assert.match(p[0].message, /Home DNS: the cluster's secondary node dns3 has been unreachable for 12 minutes — one node fewer to answer/);
|
||||
});
|
||||
test("the primary being down is critical, and says what it costs", () => {
|
||||
const p = problems(cluster(info([primaryDown()]), { unreachableSince: { "1": t(NOW - 30 * MINUTE) } }));
|
||||
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["technitium-node:7:1", "critical"]]);
|
||||
assert.match(p[0].message, /primary node dns1 has been unreachable for 30 minutes — the other nodes can take no configuration changes/);
|
||||
});
|
||||
test("when the node says when it was last seen, that is what counts; no clock at all says nothing yet", () => {
|
||||
assert.equal(problems(cluster(info([secondaryDown({ lastSeen: t(NOW - 3 * 60 * MINUTE) })]), { unreachableSince: { "3": t(NOW - MINUTE) } })).length, 1);
|
||||
assert.equal(problems(cluster(info([secondaryDown()]))).length, 0);
|
||||
});
|
||||
test("each node that is down is its own alert", () => {
|
||||
const c = cluster(info([primaryDown(), secondaryDown()]), { unreachableSince: { "1": t(NOW - HOUR), "3": t(NOW - HOUR) } });
|
||||
assert.deepEqual(problems(c).map((x) => x.key).sort(), ["technitium-node:7:1", "technitium-node:7:3"]);
|
||||
});
|
||||
test("a secondary that has not taken the configuration for three of its periods (and half an hour) is no longer a copy", () => {
|
||||
const synced = (ago: number, refresh = 900) => cluster(info([self(), node()], { configLastSynced: t(NOW - ago), configRefreshSeconds: refresh }));
|
||||
assert.deepEqual(problems(synced(40 * MINUTE)), [], "three periods of 15 minutes is 45");
|
||||
const p = problems(synced(50 * MINUTE));
|
||||
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["technitium-config:7", "warning"]]);
|
||||
assert.match(p[0].message, /has not taken the configuration from the primary for 50 minutes/);
|
||||
assert.deepEqual(problems(synced(25 * MINUTE, 60)), [], "a short period still allows half an hour");
|
||||
assert.equal(problems(synced(40 * MINUTE, 60)).length, 1, "past half an hour and three periods of a minute");
|
||||
assert.equal(problems(synced(3 * HOUR, 3600)).length, 0, "three periods of an hour is three hours");
|
||||
});
|
||||
test("the primary takes no configuration, so it has nothing to be late with; and a cluster that does not say when says nothing", () => {
|
||||
assert.deepEqual(problems(cluster(info([node({ id: 1, name: "dns1", type: "Primary", state: "Self" })], { configLastSynced: t(NOW - 5 * HOUR) }))), []);
|
||||
assert.deepEqual(problems(cluster(info([self()]))), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the stored state, the alerts, maintenance and /metrics", () => {
|
||||
let id = 0;
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "technitium", name: "Cluster DNS", baseUrl: "http://technitium-c", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
});
|
||||
const readWith = (i: ReturnType<typeof info>) => ({ ...read(), cluster: { info: i, note: null, denied: false } });
|
||||
const stored = async () => ts.storedOf((await db.select().from(technitiumState).where(eq(technitiumState.integrationId, id)))[0]);
|
||||
|
||||
test("a read stores the cluster, the clock of a node that is down carries from read to read, and it is judged from what is stored", async () => {
|
||||
const t0 = new Date(Date.now() - 20 * MINUTE);
|
||||
const c = info([node({ id: 1, name: "dns1", type: "Primary", state: "Unreachable" }), node({ id: 2, name: "dns2", type: "Secondary", state: "Self" })], { configLastSynced: t(Date.now() - 2 * 60 * MINUTE) });
|
||||
await ts.storeRead(id, readWith(c), t0);
|
||||
await ts.storeRead(id, readWith(c), new Date());
|
||||
const s = await stored();
|
||||
assert.deepEqual([s.cluster?.info?.nodes.length, s.cluster?.unreachableSince], [2, { "1": t0.toISOString() }]);
|
||||
assert.deepEqual(ts.technitiumProblems(id, "Cluster DNS", s, Date.now()).map((x) => x.key).sort(), [`technitium-config:${id}`, `technitium-node:${id}:1`]);
|
||||
});
|
||||
|
||||
test("the Alerts page files them under Services and Monitoring, and maintenance on the integration holds them", async () => {
|
||||
await setInternalFlag("technitiumSync", JSON.stringify({ [id]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const find = (key: string) => report.alerts.find((x) => x.id.endsWith(key));
|
||||
const node1 = find(`technitium-node:${id}:1`);
|
||||
assert.ok(node1, "the node alert is raised");
|
||||
assert.deepEqual([node1!.category, node1!.severity, node1!.source, node1!.link, node1!.target?.type], ["services", "critical", "Technitium DNS", "/technitium", "integration"]);
|
||||
assert.equal(find(`technitium-config:${id}`)?.category, "monitoring");
|
||||
assert.equal(subjectOfConditionKey(`technitium-node:${id}:1`), `integration:${id}`);
|
||||
assert.equal(subjectOfConditionKey(`technitium-config:${id}`), `integration:${id}`);
|
||||
});
|
||||
|
||||
test("/metrics has how many nodes there are, how many cannot be reached, and when this one last synced", async () => {
|
||||
const text = await buildMetrics(new Date());
|
||||
assert.match(text, /homelab_technitium_cluster_nodes\{name="Cluster DNS"\} 2/);
|
||||
assert.match(text, /homelab_technitium_cluster_nodes_unreachable\{name="Cluster DNS"\} 1/);
|
||||
assert.match(text, /homelab_technitium_cluster_config_synced_timestamp_seconds\{name="Cluster DNS"\} 1\d{9}/);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("the health pass, the Alerts page, the page's route and /metrics", () => {
|
||||
let tId = 0;
|
||||
let app: Server;
|
||||
|
||||
Reference in new issue
Block a user