Show cluster health in the Technitium DNS integration

Reads /api/admin/cluster/state (Technitium 14+): a Cluster table on the page,
alerts for a node unreachable for ten minutes (primary critical) and for a
secondary that has stopped taking the primary's configuration, and three
/metrics series. A server in no cluster shows nothing; a token that may not
view administration is told so and the rest of the read is unaffected.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-11-08 23:37:11 +01:00
1 parent 1ef686d1ba
commit d78aba8bf9
15 files changed
+9423 -15

No files matched your search

+2 -2
View File
@@ -144,8 +144,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "versions", id: Number(m[1]), category: "updates" };
m = /^adguard-(stopped|dns|protection|filters):(\d+)/.exec(key);
if (m) return { type: "adguard", id: Number(m[2]), category: m[1] === "stopped" || m[1] === "dns" ? "services" : "monitoring" };
m = /^technitium-(dns|blocking|lists|zones):(\d+)/.exec(key);
if (m) return { type: "technitium", id: Number(m[2]), category: m[1] === "dns" || m[1] === "zones" ? "services" : "monitoring" };
m = /^technitium-(dns|blocking|lists|zones|node|config):(\d+)/.exec(key);
if (m) return { type: "technitium", id: Number(m[2]), category: m[1] === "dns" || m[1] === "zones" || m[1] === "node" ? "services" : "monitoring" };
m = /^grafana-(firing|rule-error|ds):(\d+):/.exec(key);
if (m) return { type: "grafana", id: Number(m[2]), category: m[1] === "ds" ? "integration" : "monitoring" };
m = /^(sonarr|radarr|prowlarr|lidarr|readarr)-(health|queue|disk):(\d+)/.exec(key);
+1 -1
View File
@@ -53,7 +53,7 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `integration:${m[1]}`;
m = /^adguard-(?:stopped|dns|protection|filters):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^technitium-(?:dns|blocking|lists|zones):(\d+)/.exec(key);
m = /^technitium-(?:dns|blocking|lists|zones|node|config):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^grafana-(?:firing|rule-error|ds):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
+6
View File
@@ -150,6 +150,12 @@ async function addTechnitiumMetrics(out: Exposition, now: Date) {
out.add("homelab_technitium_zone_problems", "gauge", "Zones that have expired or cannot sync.", state.zones.issues.length, labels);
}
if (state.dhcp) out.add("homelab_technitium_dhcp_leases", "gauge", "DHCP leases Technitium holds.", state.dhcp.leases, labels);
const cluster = state.cluster?.info;
if (cluster) {
out.add("homelab_technitium_cluster_nodes", "gauge", "Nodes in the cluster Technitium belongs to.", cluster.nodes.length, labels);
out.add("homelab_technitium_cluster_nodes_unreachable", "gauge", "Of them, the ones this node cannot reach.", cluster.nodes.filter((n) => n.state.toLowerCase() === "unreachable").length, labels);
if (cluster.configLastSynced !== null) out.add("homelab_technitium_cluster_config_synced_timestamp_seconds", "gauge", "When this node last took its configuration from the cluster's primary, as a Unix time (a secondary only).", Math.floor(Date.parse(cluster.configLastSynced) / 1000), labels);
}
}
}
+64 -2
View File
@@ -5,13 +5,14 @@
*
* What is raised: **DNS is not answering** — the test question went unanswered, or was answered with an error, in every read for ten minutes (critical; one lost packet is not an outage);
* **blocking is switched off** for an hour or more while block lists are set up (someone paused it and forgot: warning); **the block lists are overdue** — their next update was due more than
* six hours ago (warning); **a zone has a problem** — a secondary zone that has expired no longer answers (critical), one that cannot sync is going stale (warning). "Since when" is what this
* six hours ago (warning); **a zone has a problem** — a secondary zone that has expired no longer answers (critical), one that cannot sync is going stale (warning); **a cluster node is unreachable**
* for ten minutes (the primary: critical, a secondary: warning); **a secondary has not synced its configuration** from the primary for much longer than it should (warning). "Since when" is what this
* app saw: the time of the first read that showed it.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, technitiumState } from "../db/schema.js";
import { createTechnitiumAdapter, type TechnitiumDhcp, type TechnitiumRead, type TechnitiumStats, type TechnitiumZones } from "../integrations/technitium/adapter.js";
import { createTechnitiumAdapter, type TechnitiumCluster, type TechnitiumDhcp, type TechnitiumRead, type TechnitiumStats, type TechnitiumZones } from "../integrations/technitium/adapter.js";
import type { ProbeResult } from "../integrations/adguard/dnsProbe.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
@@ -29,6 +30,11 @@ export const DNS_FAIL_AFTER_MS = 10 * MIN;
export const BLOCKING_OFF_AFTER_MS = HOUR;
/** Block lists whose next update was due this long ago have stopped updating (Technitium tries again soon after a failed one). */
export const LISTS_OVERDUE_MS = 6 * HOUR;
/** A cluster node that has been unreachable for this long is down (the nodes check one another every half minute, so a blip is gone long before). */
export const NODE_DOWN_AFTER_MS = 10 * MIN;
/** A secondary that has not synced for this many of its own refresh periods (and at least half an hour) has stopped following the primary. */
export const CONFIG_STALE_PERIODS = 3;
export const CONFIG_STALE_MIN_MS = 30 * MIN;
export interface TechnitiumSyncStatus {
/** The last read that worked (kept when a later one failed). */
@@ -54,6 +60,13 @@ export const isTechnitiumFresh = (status: TechnitiumSyncStatus | undefined, nowM
export type StateRow = typeof technitiumState.$inferSelect;
/** The cluster as it is stored: what was read, why it could not be (if so), and since when each node has been seen unreachable by this app. */
export interface StoredCluster {
info: TechnitiumCluster | null;
note: string | null;
unreachableSince: Record<string, string>;
}
/** What is stored of one Technitium, as the read gave it, plus what the app has seen between reads. */
export interface StoredTechnitium {
version: string | null;
@@ -69,6 +82,8 @@ export interface StoredTechnitium {
probe: (ProbeResult & { name: string; host: string }) | null;
/** When the first read that showed the DNS test failing was made; null while it works. */
probeFailSince: string | null;
/** The cluster it belongs to; null when it was never asked (a row from before). */
cluster: StoredCluster | null;
updatedAt: string;
}
@@ -95,9 +110,27 @@ export const storedOf = (row: StateRow): StoredTechnitium => ({
dhcp: parse<TechnitiumDhcp>(row.dhcp),
probe: parse<StoredTechnitium["probe"]>(row.probe),
probeFailSince: row.probeFailSince,
cluster: parse<StoredCluster>(row.cluster),
updatedAt: row.updatedAt,
});
/** The key a node is known by between reads: its id, or its name when it has none. */
export const nodeKey = (n: { id: number | null; name: string }): string => String(n.id ?? n.name);
/** Pure: the cluster to store: what was read; or, when this read failed (and was not a refusal), what was known before, with the note; and since when each node has been unreachable (first seen so, now). */
export function carryCluster(read: TechnitiumRead["cluster"], previous: StoredCluster | null, now: Date): StoredCluster | null {
if (read === undefined) return previous;
const info = read.note !== null && !read.denied && previous?.info ? previous.info : read.info;
const at = now.toISOString();
const unreachableSince: Record<string, string> = {};
if (read.note === null) {
for (const n of info?.nodes ?? []) if (n.state.toLowerCase() === "unreachable") unreachableSince[nodeKey(n)] = previous?.unreachableSince[nodeKey(n)] ?? at;
} else {
Object.assign(unreachableSince, previous?.unreachableSince ?? {});
}
return { info, note: read.note, unreachableSince };
}
/** Pure: what the app has seen between reads, carried from the last read. A thing first seen wrong starts its clock now. */
export function carrySince(read: Pick<TechnitiumRead, "blockingEnabled" | "probe">, previous: Pick<StoredTechnitium, "blockingOffSince" | "probeFailSince"> | null, now: Date): { blockingOffSince: string | null; probeFailSince: string | null } {
const at = now.toISOString();
@@ -123,6 +156,7 @@ export async function storeRead(integrationId: number, r: TechnitiumRead, now: D
dhcp: r.dhcp && JSON.stringify(r.dhcp),
probe: JSON.stringify(r.probe),
probeFailSince: since.probeFailSince,
cluster: JSON.stringify(carryCluster(r.cluster, previous ? storedOf(previous).cluster : null, now)),
updatedAt: now.toISOString(),
};
await db.insert(technitiumState).values({ integrationId, ...values }).onConflictDoUpdate({ target: technitiumState.integrationId, set: values });
@@ -186,6 +220,33 @@ export function blockListsOverdueMs(s: Pick<StoredTechnitium, "settings">, nowMs
return over > LISTS_OVERDUE_MS ? over : null;
}
/** Pure: what is wrong with the cluster: a node it cannot reach (for long enough), a secondary that no longer follows the primary. Nothing is said while the cluster could not be read. */
export function clusterProblems(id: number, name: string, c: StoredCluster | null, nowMs: number): TechnitiumProblem[] {
const out: TechnitiumProblem[] = [];
if (!c || !c.info || c.note !== null) return out;
for (const n of c.info.nodes) {
if (n.state.toLowerCase() !== "unreachable") continue;
// The node's own last contact if it says; else since this app first saw it so.
const since = n.lastSeen ?? c.unreachableSince[nodeKey(n)] ?? null;
if (since === null || nowMs - Date.parse(since) < NODE_DOWN_AFTER_MS) continue;
const primary = n.type.toLowerCase() === "primary";
out.push({
key: `technitium-node:${id}:${nodeKey(n)}`,
severity: primary ? "critical" : "warning",
message: `${name}: the cluster's ${primary ? "primary" : "secondary"} node ${n.name} has been unreachable for ${ago(nowMs - Date.parse(since))}${primary ? " — the other nodes can take no configuration changes while it is down" : " — one node fewer to answer"}`,
});
}
const self = c.info.nodes.find((n) => n.state.toLowerCase() === "self");
if (self && self.type.toLowerCase() === "secondary" && c.info.configLastSynced !== null) {
const limit = Math.max(CONFIG_STALE_MIN_MS, CONFIG_STALE_PERIODS * (c.info.configRefreshSeconds ?? 900) * 1000);
const age = nowMs - Date.parse(c.info.configLastSynced);
if (age > limit) {
out.push({ key: `technitium-config:${id}`, severity: "warning", message: `${name}: this secondary node has not taken the configuration from the primary for ${ago(age)} — it is no longer a copy of it` });
}
}
return out;
}
/** Pure: what is wrong with one Technitium, from what is stored for it. */
export function technitiumProblems(id: number, name: string, s: StoredTechnitium, nowMs: number): TechnitiumProblem[] {
const out: TechnitiumProblem[] = [];
@@ -199,6 +260,7 @@ export function technitiumProblems(id: number, name: string, s: StoredTechnitium
if (overdue !== null) {
out.push({ key: `technitium-lists:${id}`, severity: "warning", message: `${name}: the block lists were due for an update ${ago(overdue)} ago and have not been updated` });
}
out.push(...clusterProblems(id, name, s.cluster, nowMs));
const issues = s.zones?.issues ?? [];
if (issues.length > 0) {
const expired = issues.some((i) => i.problem === "expired");