Show cluster health in the Technitium DNS integration
Reads /api/admin/cluster/state (Technitium 14+): a Cluster table on the page, alerts for a node unreachable for ten minutes (primary critical) and for a secondary that has stopped taking the primary's configuration, and three /metrics series. A server in no cluster shows nothing; a token that may not view administration is told so and the rest of the read is unaffected. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1ef686d1ba
commit
d78aba8bf9
15 files changed
+9423
-15
No files matched your search
@@ -0,0 +1 @@
|
||||
ALTER TABLE `technitium_state` ADD `cluster` text;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -610,6 +610,13 @@
|
||||
"when": 1794155585495,
|
||||
"tag": "0086_talented_starfox",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 87,
|
||||
"version": "6",
|
||||
"when": 1794176955959,
|
||||
"tag": "0087_majestic_praxagora",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1483,6 +1483,7 @@ export const technitiumState = sqliteTable("technitium_state", {
|
||||
dhcp: text("dhcp"), // JSON: { scopes, enabledScopes, leases }; null: no DHCP server, or could not be read
|
||||
probe: text("probe"), // JSON: the DNS question — { ok, ms, error, name, host }
|
||||
probeFailSince: text("probe_fail_since"), // ISO: the first read in which the question went unanswered; null while it is answered
|
||||
cluster: text("cluster"), // JSON: { info (the cluster's nodes and when it last synced, or null), note, unreachableSince: { node: ISO } }; null: never asked
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
|
||||
@@ -166,7 +166,7 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
|
||||
],
|
||||
technitium: [
|
||||
{ key: "url", label: "Technitium web console URL", secret: false, placeholder: "http://dns.example.lan:5380" },
|
||||
{ key: "token", label: "API token", secret: true, type: "password", placeholder: "Administration → Sessions → Create Token (a user who may view the dashboard, settings and zones)" },
|
||||
{ key: "token", label: "API token", secret: true, type: "password", placeholder: "Administration → Sessions → Create Token (a user who may view the dashboard, settings and zones — and administration, to see a cluster)" },
|
||||
{ key: "dnsHost", label: "DNS server address (optional)", secret: false, optional: true, placeholder: "the host of the URL — only if its DNS server is somewhere else" },
|
||||
{ key: "probeDomain", label: "Name to test DNS with (optional)", secret: false, optional: true, placeholder: "example.com" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
|
||||
@@ -6,7 +6,9 @@
|
||||
* Reading only: what the server is and since when it has run, whether blocking is on and when its block lists are next updated (`/api/settings/get`), what it has done in the last day — queries and
|
||||
* how each ended, cache and zone counts, the clients and names asked about most (`/api/dashboard/stats/get`), the health of its zones — a secondary zone that has expired or cannot sync
|
||||
* (`/api/zones/list`), its DHCP scopes and leases (`/api/dhcp/…`) — and one real DNS question put to its DNS server over UDP (see ../adguard/dnsProbe.ts), because the console can be up while DNS is
|
||||
* not. Technitium's API says nothing about the CPU, memory or disk of the machine it runs on: that is what a server agent on that machine reports. Nothing is changed.
|
||||
* not. And, from version 14, the state of its **cluster** (`/api/admin/cluster/state`: the nodes, which is primary, which of them it can reach, when this one last synced its configuration from the primary) —
|
||||
* which needs the token's user to be allowed to view administration; without that the rest is read as before and the page says what is missing. Technitium's API says nothing about the CPU, memory or disk
|
||||
* of the machine it runs on: that is what a server agent on that machine reports. Nothing is changed.
|
||||
*
|
||||
* Endpoints and fields are from Technitium's published API documentation (APIDOCS.md) and have not been checked against a live install here; every field is read defensively. The token goes in an
|
||||
* `Authorization: Bearer` header, which is what version 15.0 and later require.
|
||||
@@ -89,6 +91,43 @@ export interface TechnitiumDhcp {
|
||||
leases: number;
|
||||
}
|
||||
|
||||
/** One node of a cluster, as the node that was asked sees it. */
|
||||
export interface TechnitiumClusterNode {
|
||||
id: number | null;
|
||||
name: string;
|
||||
/** "Primary" or "Secondary". */
|
||||
type: string;
|
||||
/** "Self" (the node that was asked), "Connected" or "Unreachable". */
|
||||
state: string;
|
||||
url: string | null;
|
||||
version: string | null;
|
||||
/** ISO; null for never, or when it did not say. */
|
||||
lastSeen: string | null;
|
||||
ipAddresses: string[];
|
||||
}
|
||||
|
||||
/** The cluster the Technitium belongs to (version 14 and later). */
|
||||
export interface TechnitiumCluster {
|
||||
domain: string | null;
|
||||
/** The name of the node that was asked. */
|
||||
thisNode: string | null;
|
||||
heartbeatSeconds: number | null;
|
||||
configRefreshSeconds: number | null;
|
||||
/** ISO: when the node that was asked last took the configuration from the primary (a secondary does; the primary does not). */
|
||||
configLastSynced: string | null;
|
||||
nodes: TechnitiumClusterNode[];
|
||||
}
|
||||
|
||||
/** What came of asking about the cluster: its state, or why there is none to show. */
|
||||
export interface TechnitiumClusterRead {
|
||||
/** Null: not part of a cluster (or the server is older than 14), or it could not be read (then `note` says so). */
|
||||
info: TechnitiumCluster | null;
|
||||
/** Why the cluster could not be read; null when it could, or when there is none. */
|
||||
note: string | null;
|
||||
/** The note is about the token not being allowed to see it (as against a read that failed this time). */
|
||||
denied: boolean;
|
||||
}
|
||||
|
||||
export interface TechnitiumRead {
|
||||
version: string | null;
|
||||
startedAt: string | null;
|
||||
@@ -103,6 +142,8 @@ export interface TechnitiumRead {
|
||||
dhcp: TechnitiumDhcp | null;
|
||||
/** The real DNS question: whether it was answered, how fast, and about what. */
|
||||
probe: ProbeResult & { name: string; host: string };
|
||||
/** The cluster it is part of. Absent from a read made before this was asked. */
|
||||
cluster?: TechnitiumClusterRead;
|
||||
}
|
||||
|
||||
export interface TechnitiumAdapter {
|
||||
@@ -212,6 +253,29 @@ export function parseDhcp(scopesRaw: unknown, leasesRaw: unknown | null): Techni
|
||||
return { scopes: s.scopes.length, enabledScopes: s.scopes.filter((x) => obj(x).enabled === true).length, leases: leasesRaw === null ? 0 : arr(obj(leasesRaw).leases).length };
|
||||
}
|
||||
|
||||
/** Pure: `/api/admin/cluster/state`'s `response`. Null when the server is not part of a cluster. Nodes come primary first, then by name. */
|
||||
export function parseCluster(raw: unknown): TechnitiumCluster | null {
|
||||
const r = obj(raw);
|
||||
if (r.clusterInitialized !== true) return null;
|
||||
const nodes: TechnitiumClusterNode[] = [];
|
||||
for (const item of arr(r.clusterNodes)) {
|
||||
const n = obj(item);
|
||||
const name = str(n.name);
|
||||
if (name === null) continue;
|
||||
const ips = [...arr(n.ipAddresses), n.ipAddress].map(str).filter((x): x is string => x !== null);
|
||||
nodes.push({ id: num(n.id), name, type: str(n.type) ?? "Unknown", state: str(n.state) ?? "Unknown", url: str(n.url), version: str(n.version), lastSeen: isoOf(n.lastSeen), ipAddresses: [...new Set(ips)] });
|
||||
}
|
||||
nodes.sort((a, b) => Number(b.type.toLowerCase() === "primary") - Number(a.type.toLowerCase() === "primary") || a.name.localeCompare(b.name));
|
||||
return {
|
||||
domain: str(r.clusterDomain),
|
||||
thisNode: str(r.dnsServerDomain),
|
||||
heartbeatSeconds: num(r.heartbeatRefreshIntervalSeconds),
|
||||
configRefreshSeconds: num(r.configRefreshIntervalSeconds),
|
||||
configLastSynced: isoOf(r.configLastSynced),
|
||||
nodes,
|
||||
};
|
||||
}
|
||||
|
||||
export function createTechnitiumAdapter(config: TechnitiumConfig): TechnitiumAdapter {
|
||||
const raw = String(config.url ?? "").trim().replace(/\/+$/, "");
|
||||
const insecure = config.insecure === true;
|
||||
@@ -258,16 +322,29 @@ export function createTechnitiumAdapter(config: TechnitiumConfig): TechnitiumAda
|
||||
return parseDhcp(scopes, await get("/api/dhcp/leases/list"));
|
||||
}
|
||||
|
||||
/** The cluster: a server older than 14 has no such call (404) and is simply not in one; a token that may not view administration is told so, and anything else is a read that failed this time. */
|
||||
async function readCluster(): Promise<TechnitiumClusterRead> {
|
||||
try {
|
||||
return { info: parseCluster(await get("/api/admin/cluster/state")), note: null, denied: false };
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
if (/HTTP 404/.test(message)) return { info: null, note: null, denied: false };
|
||||
if (/did not accept the token|denied|HTTP 40[13]/i.test(message)) return { info: null, note: "The token's user is not allowed to view the cluster — give it Administration → View to see it here.", denied: true };
|
||||
return { info: null, note: `The cluster could not be read this time: ${message}`, denied: false };
|
||||
}
|
||||
}
|
||||
|
||||
async function read(): Promise<TechnitiumRead> {
|
||||
// The settings must work: they say it is there and the token is accepted.
|
||||
const settings = parseSettings(await get("/api/settings/get"));
|
||||
const name = str(config.probeDomain) ?? DEFAULT_PROBE_DOMAIN;
|
||||
const host = dnsHostOf();
|
||||
const [stats, zones, dhcp, probe] = await Promise.all([
|
||||
const [stats, zones, dhcp, probe, cluster] = await Promise.all([
|
||||
soft(async () => parseStats(await get("/api/dashboard/stats/get", { type: "LastDay" }))),
|
||||
soft(async () => parseZones(await get("/api/zones/list", { pageNumber: "1", zonesPerPage: "1000" }))),
|
||||
soft(readDhcp),
|
||||
host ? probeDns(host, settings.dnsPort, name) : Promise.resolve({ ok: false, ms: null, error: "no address to ask" } as ProbeResult),
|
||||
readCluster(),
|
||||
]);
|
||||
return {
|
||||
version: settings.version,
|
||||
@@ -279,6 +356,7 @@ export function createTechnitiumAdapter(config: TechnitiumConfig): TechnitiumAda
|
||||
zones,
|
||||
dhcp,
|
||||
probe: { ...probe, name, host },
|
||||
cluster,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -144,8 +144,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "versions", id: Number(m[1]), category: "updates" };
|
||||
m = /^adguard-(stopped|dns|protection|filters):(\d+)/.exec(key);
|
||||
if (m) return { type: "adguard", id: Number(m[2]), category: m[1] === "stopped" || m[1] === "dns" ? "services" : "monitoring" };
|
||||
m = /^technitium-(dns|blocking|lists|zones):(\d+)/.exec(key);
|
||||
if (m) return { type: "technitium", id: Number(m[2]), category: m[1] === "dns" || m[1] === "zones" ? "services" : "monitoring" };
|
||||
m = /^technitium-(dns|blocking|lists|zones|node|config):(\d+)/.exec(key);
|
||||
if (m) return { type: "technitium", id: Number(m[2]), category: m[1] === "dns" || m[1] === "zones" || m[1] === "node" ? "services" : "monitoring" };
|
||||
m = /^grafana-(firing|rule-error|ds):(\d+):/.exec(key);
|
||||
if (m) return { type: "grafana", id: Number(m[2]), category: m[1] === "ds" ? "integration" : "monitoring" };
|
||||
m = /^(sonarr|radarr|prowlarr|lidarr|readarr)-(health|queue|disk):(\d+)/.exec(key);
|
||||
|
||||
@@ -53,7 +53,7 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^adguard-(?:stopped|dns|protection|filters):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^technitium-(?:dns|blocking|lists|zones):(\d+)/.exec(key);
|
||||
m = /^technitium-(?:dns|blocking|lists|zones|node|config):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^grafana-(?:firing|rule-error|ds):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
|
||||
@@ -150,6 +150,12 @@ async function addTechnitiumMetrics(out: Exposition, now: Date) {
|
||||
out.add("homelab_technitium_zone_problems", "gauge", "Zones that have expired or cannot sync.", state.zones.issues.length, labels);
|
||||
}
|
||||
if (state.dhcp) out.add("homelab_technitium_dhcp_leases", "gauge", "DHCP leases Technitium holds.", state.dhcp.leases, labels);
|
||||
const cluster = state.cluster?.info;
|
||||
if (cluster) {
|
||||
out.add("homelab_technitium_cluster_nodes", "gauge", "Nodes in the cluster Technitium belongs to.", cluster.nodes.length, labels);
|
||||
out.add("homelab_technitium_cluster_nodes_unreachable", "gauge", "Of them, the ones this node cannot reach.", cluster.nodes.filter((n) => n.state.toLowerCase() === "unreachable").length, labels);
|
||||
if (cluster.configLastSynced !== null) out.add("homelab_technitium_cluster_config_synced_timestamp_seconds", "gauge", "When this node last took its configuration from the cluster's primary, as a Unix time (a secondary only).", Math.floor(Date.parse(cluster.configLastSynced) / 1000), labels);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,13 +5,14 @@
|
||||
*
|
||||
* What is raised: **DNS is not answering** — the test question went unanswered, or was answered with an error, in every read for ten minutes (critical; one lost packet is not an outage);
|
||||
* **blocking is switched off** for an hour or more while block lists are set up (someone paused it and forgot: warning); **the block lists are overdue** — their next update was due more than
|
||||
* six hours ago (warning); **a zone has a problem** — a secondary zone that has expired no longer answers (critical), one that cannot sync is going stale (warning). "Since when" is what this
|
||||
* six hours ago (warning); **a zone has a problem** — a secondary zone that has expired no longer answers (critical), one that cannot sync is going stale (warning); **a cluster node is unreachable**
|
||||
* for ten minutes (the primary: critical, a secondary: warning); **a secondary has not synced its configuration** from the primary for much longer than it should (warning). "Since when" is what this
|
||||
* app saw: the time of the first read that showed it.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, technitiumState } from "../db/schema.js";
|
||||
import { createTechnitiumAdapter, type TechnitiumDhcp, type TechnitiumRead, type TechnitiumStats, type TechnitiumZones } from "../integrations/technitium/adapter.js";
|
||||
import { createTechnitiumAdapter, type TechnitiumCluster, type TechnitiumDhcp, type TechnitiumRead, type TechnitiumStats, type TechnitiumZones } from "../integrations/technitium/adapter.js";
|
||||
import type { ProbeResult } from "../integrations/adguard/dnsProbe.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
@@ -29,6 +30,11 @@ export const DNS_FAIL_AFTER_MS = 10 * MIN;
|
||||
export const BLOCKING_OFF_AFTER_MS = HOUR;
|
||||
/** Block lists whose next update was due this long ago have stopped updating (Technitium tries again soon after a failed one). */
|
||||
export const LISTS_OVERDUE_MS = 6 * HOUR;
|
||||
/** A cluster node that has been unreachable for this long is down (the nodes check one another every half minute, so a blip is gone long before). */
|
||||
export const NODE_DOWN_AFTER_MS = 10 * MIN;
|
||||
/** A secondary that has not synced for this many of its own refresh periods (and at least half an hour) has stopped following the primary. */
|
||||
export const CONFIG_STALE_PERIODS = 3;
|
||||
export const CONFIG_STALE_MIN_MS = 30 * MIN;
|
||||
|
||||
export interface TechnitiumSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
@@ -54,6 +60,13 @@ export const isTechnitiumFresh = (status: TechnitiumSyncStatus | undefined, nowM
|
||||
|
||||
export type StateRow = typeof technitiumState.$inferSelect;
|
||||
|
||||
/** The cluster as it is stored: what was read, why it could not be (if so), and since when each node has been seen unreachable by this app. */
|
||||
export interface StoredCluster {
|
||||
info: TechnitiumCluster | null;
|
||||
note: string | null;
|
||||
unreachableSince: Record<string, string>;
|
||||
}
|
||||
|
||||
/** What is stored of one Technitium, as the read gave it, plus what the app has seen between reads. */
|
||||
export interface StoredTechnitium {
|
||||
version: string | null;
|
||||
@@ -69,6 +82,8 @@ export interface StoredTechnitium {
|
||||
probe: (ProbeResult & { name: string; host: string }) | null;
|
||||
/** When the first read that showed the DNS test failing was made; null while it works. */
|
||||
probeFailSince: string | null;
|
||||
/** The cluster it belongs to; null when it was never asked (a row from before). */
|
||||
cluster: StoredCluster | null;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
@@ -95,9 +110,27 @@ export const storedOf = (row: StateRow): StoredTechnitium => ({
|
||||
dhcp: parse<TechnitiumDhcp>(row.dhcp),
|
||||
probe: parse<StoredTechnitium["probe"]>(row.probe),
|
||||
probeFailSince: row.probeFailSince,
|
||||
cluster: parse<StoredCluster>(row.cluster),
|
||||
updatedAt: row.updatedAt,
|
||||
});
|
||||
|
||||
/** The key a node is known by between reads: its id, or its name when it has none. */
|
||||
export const nodeKey = (n: { id: number | null; name: string }): string => String(n.id ?? n.name);
|
||||
|
||||
/** Pure: the cluster to store: what was read; or, when this read failed (and was not a refusal), what was known before, with the note; and since when each node has been unreachable (first seen so, now). */
|
||||
export function carryCluster(read: TechnitiumRead["cluster"], previous: StoredCluster | null, now: Date): StoredCluster | null {
|
||||
if (read === undefined) return previous;
|
||||
const info = read.note !== null && !read.denied && previous?.info ? previous.info : read.info;
|
||||
const at = now.toISOString();
|
||||
const unreachableSince: Record<string, string> = {};
|
||||
if (read.note === null) {
|
||||
for (const n of info?.nodes ?? []) if (n.state.toLowerCase() === "unreachable") unreachableSince[nodeKey(n)] = previous?.unreachableSince[nodeKey(n)] ?? at;
|
||||
} else {
|
||||
Object.assign(unreachableSince, previous?.unreachableSince ?? {});
|
||||
}
|
||||
return { info, note: read.note, unreachableSince };
|
||||
}
|
||||
|
||||
/** Pure: what the app has seen between reads, carried from the last read. A thing first seen wrong starts its clock now. */
|
||||
export function carrySince(read: Pick<TechnitiumRead, "blockingEnabled" | "probe">, previous: Pick<StoredTechnitium, "blockingOffSince" | "probeFailSince"> | null, now: Date): { blockingOffSince: string | null; probeFailSince: string | null } {
|
||||
const at = now.toISOString();
|
||||
@@ -123,6 +156,7 @@ export async function storeRead(integrationId: number, r: TechnitiumRead, now: D
|
||||
dhcp: r.dhcp && JSON.stringify(r.dhcp),
|
||||
probe: JSON.stringify(r.probe),
|
||||
probeFailSince: since.probeFailSince,
|
||||
cluster: JSON.stringify(carryCluster(r.cluster, previous ? storedOf(previous).cluster : null, now)),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(technitiumState).values({ integrationId, ...values }).onConflictDoUpdate({ target: technitiumState.integrationId, set: values });
|
||||
@@ -186,6 +220,33 @@ export function blockListsOverdueMs(s: Pick<StoredTechnitium, "settings">, nowMs
|
||||
return over > LISTS_OVERDUE_MS ? over : null;
|
||||
}
|
||||
|
||||
/** Pure: what is wrong with the cluster: a node it cannot reach (for long enough), a secondary that no longer follows the primary. Nothing is said while the cluster could not be read. */
|
||||
export function clusterProblems(id: number, name: string, c: StoredCluster | null, nowMs: number): TechnitiumProblem[] {
|
||||
const out: TechnitiumProblem[] = [];
|
||||
if (!c || !c.info || c.note !== null) return out;
|
||||
for (const n of c.info.nodes) {
|
||||
if (n.state.toLowerCase() !== "unreachable") continue;
|
||||
// The node's own last contact if it says; else since this app first saw it so.
|
||||
const since = n.lastSeen ?? c.unreachableSince[nodeKey(n)] ?? null;
|
||||
if (since === null || nowMs - Date.parse(since) < NODE_DOWN_AFTER_MS) continue;
|
||||
const primary = n.type.toLowerCase() === "primary";
|
||||
out.push({
|
||||
key: `technitium-node:${id}:${nodeKey(n)}`,
|
||||
severity: primary ? "critical" : "warning",
|
||||
message: `${name}: the cluster's ${primary ? "primary" : "secondary"} node ${n.name} has been unreachable for ${ago(nowMs - Date.parse(since))}${primary ? " — the other nodes can take no configuration changes while it is down" : " — one node fewer to answer"}`,
|
||||
});
|
||||
}
|
||||
const self = c.info.nodes.find((n) => n.state.toLowerCase() === "self");
|
||||
if (self && self.type.toLowerCase() === "secondary" && c.info.configLastSynced !== null) {
|
||||
const limit = Math.max(CONFIG_STALE_MIN_MS, CONFIG_STALE_PERIODS * (c.info.configRefreshSeconds ?? 900) * 1000);
|
||||
const age = nowMs - Date.parse(c.info.configLastSynced);
|
||||
if (age > limit) {
|
||||
out.push({ key: `technitium-config:${id}`, severity: "warning", message: `${name}: this secondary node has not taken the configuration from the primary for ${ago(age)} — it is no longer a copy of it` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: what is wrong with one Technitium, from what is stored for it. */
|
||||
export function technitiumProblems(id: number, name: string, s: StoredTechnitium, nowMs: number): TechnitiumProblem[] {
|
||||
const out: TechnitiumProblem[] = [];
|
||||
@@ -199,6 +260,7 @@ export function technitiumProblems(id: number, name: string, s: StoredTechnitium
|
||||
if (overdue !== null) {
|
||||
out.push({ key: `technitium-lists:${id}`, severity: "warning", message: `${name}: the block lists were due for an update ${ago(overdue)} ago and have not been updated` });
|
||||
}
|
||||
out.push(...clusterProblems(id, name, s.cluster, nowMs));
|
||||
const issues = s.zones?.issues ?? [];
|
||||
if (issues.length > 0) {
|
||||
const expired = issues.some((i) => i.problem === "expired");
|
||||
|
||||
@@ -160,6 +160,9 @@ describe("the adapter against a Technitium that answers", () => {
|
||||
let dhcpScopes: unknown[] = [{ name: "lan", enabled: true }];
|
||||
/** The port the fake settings say the DNS server is on; the one of the answering fake DNS server unless a test points it elsewhere. */
|
||||
let endpointPort: number | null = null;
|
||||
/** What the fake cluster call answers: nothing (a server with no such call), a refusal of the token's user, a failure, or a cluster. */
|
||||
let clusterMode: "none" | "denied" | "boom" | "cluster" = "none";
|
||||
const clusterBody = { clusterInitialized: true, dnsServerDomain: "dns2.example.com", version: "15.6", clusterDomain: "example.com", heartbeatRefreshIntervalSeconds: 30, configRefreshIntervalSeconds: 900, configLastSynced: "2026-09-26T12:30:16.1234567Z", clusterNodes: [{ id: 2, name: "dns2.example.com", url: "https://dns2.example.com:53443/", ipAddress: "10.0.0.3", type: "Secondary", state: "Self", version: "15.6" }, { id: 3, name: "dns3.example.com", type: "Secondary", state: "Unreachable", lastSeen: "0001-01-01T00:00:00" }, { id: 1, name: "dns1.example.com", url: "https://dns1.example.com:53443/", ipAddresses: ["10.0.0.2"], type: "Primary", state: "Connected", version: "15.6", lastSeen: "2026-09-26T12:30:16Z" }] };
|
||||
|
||||
before(async () => {
|
||||
dns = await fakeDns(() => ({}));
|
||||
@@ -186,6 +189,11 @@ describe("the adapter against a Technitium that answers", () => {
|
||||
return ok({ scopes: dhcpScopes });
|
||||
case "/api/dhcp/leases/list":
|
||||
return ok({ leases: [{}, {}] });
|
||||
case "/api/admin/cluster/state":
|
||||
if (clusterMode === "denied") return send(200, JSON.stringify({ status: "error", errorMessage: "Access was denied." }));
|
||||
if (clusterMode === "boom") return send(500, "{}");
|
||||
if (clusterMode === "cluster") return ok(clusterBody);
|
||||
return send(404, "{}");
|
||||
default:
|
||||
return send(404, "{}");
|
||||
}
|
||||
@@ -205,13 +213,14 @@ describe("the adapter against a Technitium that answers", () => {
|
||||
httpStatus = 200;
|
||||
rawBody = null;
|
||||
dhcpScopes = [{ name: "lan", enabled: true }];
|
||||
clusterMode = "none";
|
||||
};
|
||||
|
||||
test("a read puts the token in the Authorization header, asks the five things and one DNS question, and says what each answered", async () => {
|
||||
test("a read puts the token in the Authorization header, asks the six things and one DNS question, and says what each answered", async () => {
|
||||
reset();
|
||||
const r = await adapter().read();
|
||||
assert.deepEqual([...new Set(seen.map((s) => s.auth))], ["Bearer tok-1"]);
|
||||
assert.deepEqual(seen.map((s) => s.path).sort(), ["/api/dashboard/stats/get?type=LastDay", "/api/dhcp/leases/list", "/api/dhcp/scopes/list", "/api/settings/get", "/api/zones/list?pageNumber=1&zonesPerPage=1000"]);
|
||||
assert.deepEqual(seen.map((s) => s.path).sort(), ["/api/admin/cluster/state", "/api/dashboard/stats/get?type=LastDay", "/api/dhcp/leases/list", "/api/dhcp/scopes/list", "/api/settings/get", "/api/zones/list?pageNumber=1&zonesPerPage=1000"]);
|
||||
assert.deepEqual([r.version, r.dnsPort, r.blockingEnabled, r.settings.blockLists.count, r.settings.cacheMaximumEntries], ["15.5", dns.port, true, 1, 10000]);
|
||||
assert.deepEqual([r.stats?.queries, r.stats?.blockedPercent, r.zones?.issues, r.dhcp], [1000, 20, [{ name: "lab.lan", problem: "expired" }], { scopes: 1, enabledScopes: 1, leases: 2 }]);
|
||||
assert.deepEqual([r.probe.ok, r.probe.name, r.probe.host], [true, "example.com", "127.0.0.1"]);
|
||||
@@ -228,6 +237,30 @@ describe("the adapter against a Technitium that answers", () => {
|
||||
assert.equal(seen.some((s) => s.path === "/api/dhcp/leases/list"), false);
|
||||
});
|
||||
|
||||
test("the cluster: a server with no such call is in none, one in a cluster says its nodes, and neither costs the rest of the read", async () => {
|
||||
reset();
|
||||
const none = await adapter().read();
|
||||
assert.deepEqual(none.cluster, { info: null, note: null, denied: false });
|
||||
clusterMode = "cluster";
|
||||
const r = await adapter().read();
|
||||
assert.equal(r.cluster?.note, null);
|
||||
assert.deepEqual([r.cluster?.info?.domain, r.cluster?.info?.thisNode, r.cluster?.info?.nodes.map((n) => `${n.name}=${n.type}/${n.state}`)], ["example.com", "dns2.example.com", ["dns1.example.com=Primary/Connected", "dns2.example.com=Secondary/Self", "dns3.example.com=Secondary/Unreachable"]]);
|
||||
assert.equal(r.stats?.queries, 1000, "the rest of the read is as before");
|
||||
});
|
||||
|
||||
test("the cluster: a token that may not view it is told so, a read that failed this time is said, and the rest of the read still works", async () => {
|
||||
reset();
|
||||
clusterMode = "denied";
|
||||
const denied = await adapter().read();
|
||||
assert.equal(denied.cluster?.denied, true);
|
||||
assert.match(denied.cluster?.note ?? "", /not allowed to view the cluster/);
|
||||
clusterMode = "boom";
|
||||
const boom = await adapter().read();
|
||||
assert.equal(boom.cluster?.denied, false);
|
||||
assert.match(boom.cluster?.note ?? "", /could not be read this time: Technitium error: HTTP 500/);
|
||||
assert.equal(boom.version, "15.5");
|
||||
});
|
||||
|
||||
test("what cannot be read is left out and the rest is kept: statistics, zones and DHCP are each soft; the settings are not", async () => {
|
||||
reset();
|
||||
failing = new Set(["/api/dashboard/stats/get", "/api/zones/list", "/api/dhcp/scopes/list"]);
|
||||
@@ -429,6 +462,151 @@ describe("what Technitium has stored", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("the cluster: reading it, carrying it, judging it", () => {
|
||||
const MINUTE = MIN;
|
||||
const t = iso;
|
||||
const NOW = Date.parse("2026-11-01T12:00:00Z");
|
||||
const node = (over: Partial<import("../src/integrations/technitium/adapter.js").TechnitiumClusterNode> = {}): import("../src/integrations/technitium/adapter.js").TechnitiumClusterNode => ({ id: 1, name: "dns1", type: "Primary", state: "Connected", url: null, version: "15.6", lastSeen: null, ipAddresses: [], ...over });
|
||||
const info = (nodes: ReturnType<typeof node>[], over: Partial<import("../src/integrations/technitium/adapter.js").TechnitiumCluster> = {}): import("../src/integrations/technitium/adapter.js").TechnitiumCluster => ({ domain: "example.com", thisNode: "dns2", heartbeatSeconds: 30, configRefreshSeconds: 900, configLastSynced: null, nodes, ...over });
|
||||
const cluster = (i: ReturnType<typeof info> | null, over: Partial<import("../src/services/technitiumSync.js").StoredCluster> = {}): import("../src/services/technitiumSync.js").StoredCluster => ({ info: i, note: null, unreachableSince: {}, ...over });
|
||||
|
||||
describe("parsing what Technitium says", () => {
|
||||
test("a server that is not in a cluster, or says nonsense, has none", () => {
|
||||
assert.equal(tn.parseCluster({ clusterInitialized: false, dnsServerDomain: "x", version: "15.6" }), null);
|
||||
for (const junk of [null, undefined, "x", [], {}, { clusterInitialized: "yes" }]) assert.equal(tn.parseCluster(junk), null, JSON.stringify(junk));
|
||||
});
|
||||
test("the nodes come primary first and then by name; a node with no name is left out; an address may be one or many, and never twice", () => {
|
||||
const c = tn.parseCluster({ clusterInitialized: true, clusterNodes: [{ name: "b", type: "Secondary", state: "Connected", ipAddress: "10.0.0.2", ipAddresses: ["10.0.0.2", "10.0.0.9"] }, { type: "Secondary" }, { name: "a", type: "Secondary", state: "Self" }, { name: "z", type: "Primary", state: "Connected" }] })!;
|
||||
assert.deepEqual(c.nodes.map((n) => n.name), ["z", "a", "b"]);
|
||||
assert.deepEqual(c.nodes[2].ipAddresses, ["10.0.0.2", "10.0.0.9"]);
|
||||
});
|
||||
test("a moment is read as Technitium writes it, and the year 1 it writes for never is none", () => {
|
||||
const c = tn.parseCluster({ clusterInitialized: true, configLastSynced: "2026-09-26T12:30:16.1234567Z", clusterNodes: [{ name: "a", lastSeen: "0001-01-01T00:00:00" }, { name: "b", lastSeen: "2026-09-26T12:30:16Z" }] })!;
|
||||
assert.equal(c.configLastSynced, "2026-09-26T12:30:16.123Z");
|
||||
assert.deepEqual(c.nodes.map((n) => [n.name, n.lastSeen]), [["a", null], ["b", "2026-09-26T12:30:16.000Z"]]);
|
||||
});
|
||||
test("what a node leaves out is unknown, not an error", () => {
|
||||
const c = tn.parseCluster({ clusterInitialized: true, clusterNodes: [{ name: "a" }] })!;
|
||||
assert.deepEqual([c.nodes[0].type, c.nodes[0].state, c.nodes[0].id, c.nodes[0].version, c.domain, c.configLastSynced], ["Unknown", "Unknown", null, null, null, null]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("carrying it between reads", () => {
|
||||
const read = (i: ReturnType<typeof info> | null, note: string | null = null, denied = false) => ({ info: i, note, denied });
|
||||
const down = info([node({ id: 3, name: "dns3", type: "Secondary", state: "Unreachable" })]);
|
||||
test("a node first seen unreachable starts its clock now, keeps it while it stays so, and loses it when it comes back", () => {
|
||||
const first = ts.carryCluster(read(down), null, new Date(NOW))!;
|
||||
assert.deepEqual(first.unreachableSince, { "3": t(NOW) });
|
||||
const later = ts.carryCluster(read(down), first, new Date(NOW + 5 * MINUTE))!;
|
||||
assert.deepEqual(later.unreachableSince, { "3": t(NOW) });
|
||||
const back = ts.carryCluster(read(info([node({ id: 3, name: "dns3", type: "Secondary" })])), later, new Date(NOW + 6 * MINUTE))!;
|
||||
assert.deepEqual(back.unreachableSince, {});
|
||||
});
|
||||
test("a node with no id is known by its name", () => {
|
||||
const c = ts.carryCluster(read(info([node({ id: null, name: "dnsX", state: "Unreachable" })])), null, new Date(NOW))!;
|
||||
assert.deepEqual(Object.keys(c.unreachableSince), ["dnsX"]);
|
||||
});
|
||||
test("a read that failed this time keeps what was known and the clocks, and says why; a refusal does not keep it", () => {
|
||||
const before = ts.carryCluster(read(down), null, new Date(NOW))!;
|
||||
const failed = ts.carryCluster(read(null, "boom"), before, new Date(NOW + 4 * MINUTE))!;
|
||||
assert.deepEqual([failed.info?.nodes.length, failed.note, failed.unreachableSince], [1, "boom", { "3": t(NOW) }]);
|
||||
const refused = ts.carryCluster(read(null, "not allowed", true), before, new Date(NOW + 4 * MINUTE))!;
|
||||
assert.deepEqual([refused.info, refused.note], [null, "not allowed"]);
|
||||
});
|
||||
test("a read made before the cluster was asked about leaves what is stored as it is", () => {
|
||||
const before = cluster(down);
|
||||
assert.equal(ts.carryCluster(undefined, before, new Date(NOW)), before);
|
||||
assert.equal(ts.carryCluster(undefined, null, new Date(NOW)), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is wrong", () => {
|
||||
const primaryDown = (extra: Partial<ReturnType<typeof node>> = {}) => node({ id: 1, name: "dns1", type: "Primary", state: "Unreachable", ...extra });
|
||||
const secondaryDown = (extra: Partial<ReturnType<typeof node>> = {}) => node({ id: 3, name: "dns3", type: "Secondary", state: "Unreachable", ...extra });
|
||||
const self = (extra: Partial<ReturnType<typeof node>> = {}) => node({ id: 2, name: "dns2", type: "Secondary", state: "Self", ...extra });
|
||||
const problems = (c: ReturnType<typeof cluster> | null, now = NOW) => ts.clusterProblems(7, "Home DNS", c, now);
|
||||
|
||||
test("nothing in a cluster that is well, in none, never asked about, or that could not be read", () => {
|
||||
assert.deepEqual(problems(cluster(info([node(), self()]))), []);
|
||||
assert.deepEqual(problems(null), []);
|
||||
assert.deepEqual(problems(cluster(null)), []);
|
||||
assert.deepEqual(problems(cluster(info([secondaryDown()]), { note: "boom", unreachableSince: { "3": t(NOW - HOUR) } })), []);
|
||||
});
|
||||
test("a node unreachable for ten minutes by this app's clock is an alert; a blip is not", () => {
|
||||
assert.deepEqual(problems(cluster(info([secondaryDown()]), { unreachableSince: { "3": t(NOW - 9 * MINUTE) } })), []);
|
||||
const p = problems(cluster(info([secondaryDown()]), { unreachableSince: { "3": t(NOW - 12 * MINUTE) } }));
|
||||
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["technitium-node:7:3", "warning"]]);
|
||||
assert.match(p[0].message, /Home DNS: the cluster's secondary node dns3 has been unreachable for 12 minutes — one node fewer to answer/);
|
||||
});
|
||||
test("the primary being down is critical, and says what it costs", () => {
|
||||
const p = problems(cluster(info([primaryDown()]), { unreachableSince: { "1": t(NOW - 30 * MINUTE) } }));
|
||||
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["technitium-node:7:1", "critical"]]);
|
||||
assert.match(p[0].message, /primary node dns1 has been unreachable for 30 minutes — the other nodes can take no configuration changes/);
|
||||
});
|
||||
test("when the node says when it was last seen, that is what counts; no clock at all says nothing yet", () => {
|
||||
assert.equal(problems(cluster(info([secondaryDown({ lastSeen: t(NOW - 3 * 60 * MINUTE) })]), { unreachableSince: { "3": t(NOW - MINUTE) } })).length, 1);
|
||||
assert.equal(problems(cluster(info([secondaryDown()]))).length, 0);
|
||||
});
|
||||
test("each node that is down is its own alert", () => {
|
||||
const c = cluster(info([primaryDown(), secondaryDown()]), { unreachableSince: { "1": t(NOW - HOUR), "3": t(NOW - HOUR) } });
|
||||
assert.deepEqual(problems(c).map((x) => x.key).sort(), ["technitium-node:7:1", "technitium-node:7:3"]);
|
||||
});
|
||||
test("a secondary that has not taken the configuration for three of its periods (and half an hour) is no longer a copy", () => {
|
||||
const synced = (ago: number, refresh = 900) => cluster(info([self(), node()], { configLastSynced: t(NOW - ago), configRefreshSeconds: refresh }));
|
||||
assert.deepEqual(problems(synced(40 * MINUTE)), [], "three periods of 15 minutes is 45");
|
||||
const p = problems(synced(50 * MINUTE));
|
||||
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["technitium-config:7", "warning"]]);
|
||||
assert.match(p[0].message, /has not taken the configuration from the primary for 50 minutes/);
|
||||
assert.deepEqual(problems(synced(25 * MINUTE, 60)), [], "a short period still allows half an hour");
|
||||
assert.equal(problems(synced(40 * MINUTE, 60)).length, 1, "past half an hour and three periods of a minute");
|
||||
assert.equal(problems(synced(3 * HOUR, 3600)).length, 0, "three periods of an hour is three hours");
|
||||
});
|
||||
test("the primary takes no configuration, so it has nothing to be late with; and a cluster that does not say when says nothing", () => {
|
||||
assert.deepEqual(problems(cluster(info([node({ id: 1, name: "dns1", type: "Primary", state: "Self" })], { configLastSynced: t(NOW - 5 * HOUR) }))), []);
|
||||
assert.deepEqual(problems(cluster(info([self()]))), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the stored state, the alerts, maintenance and /metrics", () => {
|
||||
let id = 0;
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "technitium", name: "Cluster DNS", baseUrl: "http://technitium-c", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
});
|
||||
const readWith = (i: ReturnType<typeof info>) => ({ ...read(), cluster: { info: i, note: null, denied: false } });
|
||||
const stored = async () => ts.storedOf((await db.select().from(technitiumState).where(eq(technitiumState.integrationId, id)))[0]);
|
||||
|
||||
test("a read stores the cluster, the clock of a node that is down carries from read to read, and it is judged from what is stored", async () => {
|
||||
const t0 = new Date(Date.now() - 20 * MINUTE);
|
||||
const c = info([node({ id: 1, name: "dns1", type: "Primary", state: "Unreachable" }), node({ id: 2, name: "dns2", type: "Secondary", state: "Self" })], { configLastSynced: t(Date.now() - 2 * 60 * MINUTE) });
|
||||
await ts.storeRead(id, readWith(c), t0);
|
||||
await ts.storeRead(id, readWith(c), new Date());
|
||||
const s = await stored();
|
||||
assert.deepEqual([s.cluster?.info?.nodes.length, s.cluster?.unreachableSince], [2, { "1": t0.toISOString() }]);
|
||||
assert.deepEqual(ts.technitiumProblems(id, "Cluster DNS", s, Date.now()).map((x) => x.key).sort(), [`technitium-config:${id}`, `technitium-node:${id}:1`]);
|
||||
});
|
||||
|
||||
test("the Alerts page files them under Services and Monitoring, and maintenance on the integration holds them", async () => {
|
||||
await setInternalFlag("technitiumSync", JSON.stringify({ [id]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const find = (key: string) => report.alerts.find((x) => x.id.endsWith(key));
|
||||
const node1 = find(`technitium-node:${id}:1`);
|
||||
assert.ok(node1, "the node alert is raised");
|
||||
assert.deepEqual([node1!.category, node1!.severity, node1!.source, node1!.link, node1!.target?.type], ["services", "critical", "Technitium DNS", "/technitium", "integration"]);
|
||||
assert.equal(find(`technitium-config:${id}`)?.category, "monitoring");
|
||||
assert.equal(subjectOfConditionKey(`technitium-node:${id}:1`), `integration:${id}`);
|
||||
assert.equal(subjectOfConditionKey(`technitium-config:${id}`), `integration:${id}`);
|
||||
});
|
||||
|
||||
test("/metrics has how many nodes there are, how many cannot be reached, and when this one last synced", async () => {
|
||||
const text = await buildMetrics(new Date());
|
||||
assert.match(text, /homelab_technitium_cluster_nodes\{name="Cluster DNS"\} 2/);
|
||||
assert.match(text, /homelab_technitium_cluster_nodes_unreachable\{name="Cluster DNS"\} 1/);
|
||||
assert.match(text, /homelab_technitium_cluster_config_synced_timestamp_seconds\{name="Cluster DNS"\} 1\d{9}/);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("the health pass, the Alerts page, the page's route and /metrics", () => {
|
||||
let tId = 0;
|
||||
let app: Server;
|
||||
|
||||
Reference in new issue
Block a user