Show cluster health in the Technitium DNS integration
Reads /api/admin/cluster/state (Technitium 14+): a Cluster table on the page, alerts for a node unreachable for ten minutes (primary critical) and for a secondary that has stopped taking the primary's configuration, and three /metrics series. A server in no cluster shows nothing; a token that may not view administration is told so and the rest of the read is unaffected. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1ef686d1ba
commit
d78aba8bf9
15 files changed
+9423
-15
No files matched your search
@@ -144,8 +144,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "versions", id: Number(m[1]), category: "updates" };
|
||||
m = /^adguard-(stopped|dns|protection|filters):(\d+)/.exec(key);
|
||||
if (m) return { type: "adguard", id: Number(m[2]), category: m[1] === "stopped" || m[1] === "dns" ? "services" : "monitoring" };
|
||||
m = /^technitium-(dns|blocking|lists|zones):(\d+)/.exec(key);
|
||||
if (m) return { type: "technitium", id: Number(m[2]), category: m[1] === "dns" || m[1] === "zones" ? "services" : "monitoring" };
|
||||
m = /^technitium-(dns|blocking|lists|zones|node|config):(\d+)/.exec(key);
|
||||
if (m) return { type: "technitium", id: Number(m[2]), category: m[1] === "dns" || m[1] === "zones" || m[1] === "node" ? "services" : "monitoring" };
|
||||
m = /^grafana-(firing|rule-error|ds):(\d+):/.exec(key);
|
||||
if (m) return { type: "grafana", id: Number(m[2]), category: m[1] === "ds" ? "integration" : "monitoring" };
|
||||
m = /^(sonarr|radarr|prowlarr|lidarr|readarr)-(health|queue|disk):(\d+)/.exec(key);
|
||||
|
||||
@@ -53,7 +53,7 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^adguard-(?:stopped|dns|protection|filters):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^technitium-(?:dns|blocking|lists|zones):(\d+)/.exec(key);
|
||||
m = /^technitium-(?:dns|blocking|lists|zones|node|config):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^grafana-(?:firing|rule-error|ds):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
|
||||
@@ -150,6 +150,12 @@ async function addTechnitiumMetrics(out: Exposition, now: Date) {
|
||||
out.add("homelab_technitium_zone_problems", "gauge", "Zones that have expired or cannot sync.", state.zones.issues.length, labels);
|
||||
}
|
||||
if (state.dhcp) out.add("homelab_technitium_dhcp_leases", "gauge", "DHCP leases Technitium holds.", state.dhcp.leases, labels);
|
||||
const cluster = state.cluster?.info;
|
||||
if (cluster) {
|
||||
out.add("homelab_technitium_cluster_nodes", "gauge", "Nodes in the cluster Technitium belongs to.", cluster.nodes.length, labels);
|
||||
out.add("homelab_technitium_cluster_nodes_unreachable", "gauge", "Of them, the ones this node cannot reach.", cluster.nodes.filter((n) => n.state.toLowerCase() === "unreachable").length, labels);
|
||||
if (cluster.configLastSynced !== null) out.add("homelab_technitium_cluster_config_synced_timestamp_seconds", "gauge", "When this node last took its configuration from the cluster's primary, as a Unix time (a secondary only).", Math.floor(Date.parse(cluster.configLastSynced) / 1000), labels);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,13 +5,14 @@
|
||||
*
|
||||
* What is raised: **DNS is not answering** — the test question went unanswered, or was answered with an error, in every read for ten minutes (critical; one lost packet is not an outage);
|
||||
* **blocking is switched off** for an hour or more while block lists are set up (someone paused it and forgot: warning); **the block lists are overdue** — their next update was due more than
|
||||
* six hours ago (warning); **a zone has a problem** — a secondary zone that has expired no longer answers (critical), one that cannot sync is going stale (warning). "Since when" is what this
|
||||
* six hours ago (warning); **a zone has a problem** — a secondary zone that has expired no longer answers (critical), one that cannot sync is going stale (warning); **a cluster node is unreachable**
|
||||
* for ten minutes (the primary: critical, a secondary: warning); **a secondary has not synced its configuration** from the primary for much longer than it should (warning). "Since when" is what this
|
||||
* app saw: the time of the first read that showed it.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, technitiumState } from "../db/schema.js";
|
||||
import { createTechnitiumAdapter, type TechnitiumDhcp, type TechnitiumRead, type TechnitiumStats, type TechnitiumZones } from "../integrations/technitium/adapter.js";
|
||||
import { createTechnitiumAdapter, type TechnitiumCluster, type TechnitiumDhcp, type TechnitiumRead, type TechnitiumStats, type TechnitiumZones } from "../integrations/technitium/adapter.js";
|
||||
import type { ProbeResult } from "../integrations/adguard/dnsProbe.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
@@ -29,6 +30,11 @@ export const DNS_FAIL_AFTER_MS = 10 * MIN;
|
||||
export const BLOCKING_OFF_AFTER_MS = HOUR;
|
||||
/** Block lists whose next update was due this long ago have stopped updating (Technitium tries again soon after a failed one). */
|
||||
export const LISTS_OVERDUE_MS = 6 * HOUR;
|
||||
/** A cluster node that has been unreachable for this long is down (the nodes check one another every half minute, so a blip is gone long before). */
|
||||
export const NODE_DOWN_AFTER_MS = 10 * MIN;
|
||||
/** A secondary that has not synced for this many of its own refresh periods (and at least half an hour) has stopped following the primary. */
|
||||
export const CONFIG_STALE_PERIODS = 3;
|
||||
export const CONFIG_STALE_MIN_MS = 30 * MIN;
|
||||
|
||||
export interface TechnitiumSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
@@ -54,6 +60,13 @@ export const isTechnitiumFresh = (status: TechnitiumSyncStatus | undefined, nowM
|
||||
|
||||
export type StateRow = typeof technitiumState.$inferSelect;
|
||||
|
||||
/** The cluster as it is stored: what was read, why it could not be (if so), and since when each node has been seen unreachable by this app. */
|
||||
export interface StoredCluster {
|
||||
info: TechnitiumCluster | null;
|
||||
note: string | null;
|
||||
unreachableSince: Record<string, string>;
|
||||
}
|
||||
|
||||
/** What is stored of one Technitium, as the read gave it, plus what the app has seen between reads. */
|
||||
export interface StoredTechnitium {
|
||||
version: string | null;
|
||||
@@ -69,6 +82,8 @@ export interface StoredTechnitium {
|
||||
probe: (ProbeResult & { name: string; host: string }) | null;
|
||||
/** When the first read that showed the DNS test failing was made; null while it works. */
|
||||
probeFailSince: string | null;
|
||||
/** The cluster it belongs to; null when it was never asked (a row from before). */
|
||||
cluster: StoredCluster | null;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
@@ -95,9 +110,27 @@ export const storedOf = (row: StateRow): StoredTechnitium => ({
|
||||
dhcp: parse<TechnitiumDhcp>(row.dhcp),
|
||||
probe: parse<StoredTechnitium["probe"]>(row.probe),
|
||||
probeFailSince: row.probeFailSince,
|
||||
cluster: parse<StoredCluster>(row.cluster),
|
||||
updatedAt: row.updatedAt,
|
||||
});
|
||||
|
||||
/** The key a node is known by between reads: its id, or its name when it has none. */
|
||||
export const nodeKey = (n: { id: number | null; name: string }): string => String(n.id ?? n.name);
|
||||
|
||||
/** Pure: the cluster to store: what was read; or, when this read failed (and was not a refusal), what was known before, with the note; and since when each node has been unreachable (first seen so, now). */
|
||||
export function carryCluster(read: TechnitiumRead["cluster"], previous: StoredCluster | null, now: Date): StoredCluster | null {
|
||||
if (read === undefined) return previous;
|
||||
const info = read.note !== null && !read.denied && previous?.info ? previous.info : read.info;
|
||||
const at = now.toISOString();
|
||||
const unreachableSince: Record<string, string> = {};
|
||||
if (read.note === null) {
|
||||
for (const n of info?.nodes ?? []) if (n.state.toLowerCase() === "unreachable") unreachableSince[nodeKey(n)] = previous?.unreachableSince[nodeKey(n)] ?? at;
|
||||
} else {
|
||||
Object.assign(unreachableSince, previous?.unreachableSince ?? {});
|
||||
}
|
||||
return { info, note: read.note, unreachableSince };
|
||||
}
|
||||
|
||||
/** Pure: what the app has seen between reads, carried from the last read. A thing first seen wrong starts its clock now. */
|
||||
export function carrySince(read: Pick<TechnitiumRead, "blockingEnabled" | "probe">, previous: Pick<StoredTechnitium, "blockingOffSince" | "probeFailSince"> | null, now: Date): { blockingOffSince: string | null; probeFailSince: string | null } {
|
||||
const at = now.toISOString();
|
||||
@@ -123,6 +156,7 @@ export async function storeRead(integrationId: number, r: TechnitiumRead, now: D
|
||||
dhcp: r.dhcp && JSON.stringify(r.dhcp),
|
||||
probe: JSON.stringify(r.probe),
|
||||
probeFailSince: since.probeFailSince,
|
||||
cluster: JSON.stringify(carryCluster(r.cluster, previous ? storedOf(previous).cluster : null, now)),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(technitiumState).values({ integrationId, ...values }).onConflictDoUpdate({ target: technitiumState.integrationId, set: values });
|
||||
@@ -186,6 +220,33 @@ export function blockListsOverdueMs(s: Pick<StoredTechnitium, "settings">, nowMs
|
||||
return over > LISTS_OVERDUE_MS ? over : null;
|
||||
}
|
||||
|
||||
/** Pure: what is wrong with the cluster: a node it cannot reach (for long enough), a secondary that no longer follows the primary. Nothing is said while the cluster could not be read. */
|
||||
export function clusterProblems(id: number, name: string, c: StoredCluster | null, nowMs: number): TechnitiumProblem[] {
|
||||
const out: TechnitiumProblem[] = [];
|
||||
if (!c || !c.info || c.note !== null) return out;
|
||||
for (const n of c.info.nodes) {
|
||||
if (n.state.toLowerCase() !== "unreachable") continue;
|
||||
// The node's own last contact if it says; else since this app first saw it so.
|
||||
const since = n.lastSeen ?? c.unreachableSince[nodeKey(n)] ?? null;
|
||||
if (since === null || nowMs - Date.parse(since) < NODE_DOWN_AFTER_MS) continue;
|
||||
const primary = n.type.toLowerCase() === "primary";
|
||||
out.push({
|
||||
key: `technitium-node:${id}:${nodeKey(n)}`,
|
||||
severity: primary ? "critical" : "warning",
|
||||
message: `${name}: the cluster's ${primary ? "primary" : "secondary"} node ${n.name} has been unreachable for ${ago(nowMs - Date.parse(since))}${primary ? " — the other nodes can take no configuration changes while it is down" : " — one node fewer to answer"}`,
|
||||
});
|
||||
}
|
||||
const self = c.info.nodes.find((n) => n.state.toLowerCase() === "self");
|
||||
if (self && self.type.toLowerCase() === "secondary" && c.info.configLastSynced !== null) {
|
||||
const limit = Math.max(CONFIG_STALE_MIN_MS, CONFIG_STALE_PERIODS * (c.info.configRefreshSeconds ?? 900) * 1000);
|
||||
const age = nowMs - Date.parse(c.info.configLastSynced);
|
||||
if (age > limit) {
|
||||
out.push({ key: `technitium-config:${id}`, severity: "warning", message: `${name}: this secondary node has not taken the configuration from the primary for ${ago(age)} — it is no longer a copy of it` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: what is wrong with one Technitium, from what is stored for it. */
|
||||
export function technitiumProblems(id: number, name: string, s: StoredTechnitium, nowMs: number): TechnitiumProblem[] {
|
||||
const out: TechnitiumProblem[] = [];
|
||||
@@ -199,6 +260,7 @@ export function technitiumProblems(id: number, name: string, s: StoredTechnitium
|
||||
if (overdue !== null) {
|
||||
out.push({ key: `technitium-lists:${id}`, severity: "warning", message: `${name}: the block lists were due for an update ${ago(overdue)} ago and have not been updated` });
|
||||
}
|
||||
out.push(...clusterProblems(id, name, s.cluster, nowMs));
|
||||
const issues = s.zones?.issues ?? [];
|
||||
if (issues.length > 0) {
|
||||
const expired = issues.some((i) => i.problem === "expired");
|
||||
|
||||
Reference in new issue
Block a user