/** * AdGuard Home (a real DNS question over UDP, its status, statistics and filter lists) and Grafana (alert rules and data-source health): reading what their APIs say, what is stored and * carried between reads, what is wrong, the health pass, the alerts, the pages' routes and /metrics. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import * as dgram from "node:dgram"; import { createServer as createHttpServer, type Server } from "node:http"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-adg-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, integrations, adguardState, grafanaState } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const probe = await import("../src/integrations/adguard/dnsProbe.js"); const adg = await import("../src/integrations/adguard/adapter.js"); const gfa = await import("../src/integrations/grafana/adapter.js"); const ag = await import("../src/services/adguardSync.js"); const gs = await import("../src/services/grafanaSync.js"); const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js"); const { subjectOfConditionKey } = await import("../src/services/maintenance.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { adguardRouter } = await import("../src/routes/adguard.js"); const { grafanaRouter } = await import("../src/routes/grafana.js"); const { buildMetrics } = await import("../src/services/metrics.js"); const { setInternalFlag } = await import("../src/services/settingsStore.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); const MIN = 60_000; const HOUR = 60 * MIN; const DAY = 24 * HOUR; const iso = (ms: number) => new Date(ms).toISOString(); /** A DNS server that answers with the rcode it is told to (or not at all), once per question. */ async function fakeDns(behaviour: () => { rcode?: number; silent?: boolean; wrongIdFirst?: boolean }) { const socket = dgram.createSocket("udp4"); const questions: string[] = []; socket.on("message", (msg, rinfo) => { const b = behaviour(); // The question name, to show what was asked. let i = 12; const labels: string[] = []; while (msg[i] !== 0 && i < msg.length) { labels.push(msg.subarray(i + 1, i + 1 + msg[i]).toString("ascii")); i += msg[i] + 1; } questions.push(labels.join(".")); if (b.silent) return; const reply = Buffer.from(msg); reply.writeUInt16BE(0x8180 | (b.rcode ?? 0), 2); reply.writeUInt16BE(b.rcode ? 0 : 1, 6); if (b.wrongIdFirst) { const wrong = Buffer.from(reply); wrong.writeUInt16BE((msg.readUInt16BE(0) + 1) & 0xffff, 0); socket.send(wrong, rinfo.port, rinfo.address); } socket.send(reply, rinfo.port, rinfo.address); }); await new Promise((resolve) => socket.bind(0, "127.0.0.1", resolve)); return { port: socket.address().port, questions, close: () => socket.close() }; } describe("the DNS question", () => { test("a standard recursive question for an address: one question, the name in labels", () => { const q = probe.buildQuery("example.com", 0x1234); assert.deepEqual([q.readUInt16BE(0), q.readUInt16BE(2), q.readUInt16BE(4)], [0x1234, 0x0100, 1]); assert.deepEqual([...q.subarray(12)], [7, ...Buffer.from("example"), 3, ...Buffer.from("com"), 0, 0, 1, 0, 1]); assert.throws(() => probe.buildQuery("bad name!", 1), /not a name/); assert.throws(() => probe.buildQuery("", 1), /not a name/); }); test("an answer is one with the same id and the response bit; how it ended is the rcode", () => { const q = probe.buildQuery("example.com", 7); const answer = Buffer.from(q); answer.writeUInt16BE(0x8183, 2); assert.deepEqual(probe.parseResponse(answer, 7), { rcode: 3, answers: 0 }); assert.equal(probe.parseResponse(answer, 8), null, "another question's answer"); assert.equal(probe.parseResponse(q, 7), null, "a question, not an answer"); assert.equal(probe.parseResponse(Buffer.alloc(5), 7), null); }); describe("against a DNS server that answers", () => { let behaviour: { rcode?: number; silent?: boolean; wrongIdFirst?: boolean } = {}; let dns: Awaited>; before(async () => { dns = await fakeDns(() => behaviour); }); after(() => dns.close()); test("an address, and 'no such name', both mean the resolver works; a server failure or a refusal does not", async () => { behaviour = {}; const ok = await probe.probeDns("127.0.0.1", dns.port, "example.com", 1000); assert.deepEqual([ok.ok, ok.error, typeof ok.ms], [true, null, "number"]); assert.equal(dns.questions.at(-1), "example.com"); behaviour = { rcode: 3 }; assert.equal((await probe.probeDns("127.0.0.1", dns.port, "nothere.example", 1000)).ok, true); behaviour = { rcode: 2 }; const fail = await probe.probeDns("127.0.0.1", dns.port, "example.com", 1000); assert.deepEqual([fail.ok, fail.error], [false, "answered, but with server failure (SERVFAIL)"]); behaviour = { rcode: 5 }; assert.match((await probe.probeDns("127.0.0.1", dns.port, "example.com", 1000)).error ?? "", /refused/); }); test("an answer to another question is not waited on as if it were ours; silence is a timeout", async () => { behaviour = { wrongIdFirst: true }; assert.equal((await probe.probeDns("127.0.0.1", dns.port, "example.com", 1000)).ok, true); behaviour = { silent: true }; const r = await probe.probeDns("127.0.0.1", dns.port, "example.com", 300); assert.deepEqual([r.ok, r.ms, r.error], [false, null, "no answer in 0 s"]); behaviour = {}; }); test("a name that is not one is said, not asked", async () => { assert.match((await probe.probeDns("127.0.0.1", dns.port, "not a name", 300)).error ?? "", /not a name/); }); }); }); describe("what AdGuard Home's API says", () => { test("counts come as one-name pairs, biggest first", () => { assert.deepEqual(adg.parseCounted([{ "a.com": 3 }, { "b.com": 9 }, { "c.com": "x" }, "junk"]), [{ name: "b.com", count: 9 }, { name: "a.com", count: 3 }]); assert.deepEqual(adg.parseCounted(null), []); assert.equal(adg.parseCounted(Array.from({ length: 30 }, (_, i) => ({ [`d${i}.com`]: i }))).length, 10); }); test("the statistics: queries, blocked and its share, the average time in milliseconds, the period; no query count is not zero", () => { const s = adg.parseStats({ time_units: "hours", num_dns_queries: 2000, num_blocked_filtering: 500, num_replaced_safebrowsing: 2, num_replaced_parental: 0, num_replaced_safesearch: 1, avg_processing_time: 0.0123, dns_queries: new Array(24).fill(1), top_blocked_domains: [{ "ads.example": 400 }], top_clients: [{ "192.168.1.5": 900 }], top_queried_domains: [{ "x.com": 10 }] })!; assert.deepEqual([s.queries, s.blocked, s.blockedPercent, s.avgMs, s.periodHours, s.safeBrowsing, s.safeSearch], [2000, 500, 25, 12.3, 24, 2, 1]); assert.deepEqual([s.topBlocked[0], s.topClients[0].name], [{ name: "ads.example", count: 400 }, "192.168.1.5"]); assert.equal(adg.parseStats({ time_units: "hours" }), null); assert.equal(adg.parseStats({ num_dns_queries: 0 })!.blockedPercent, null); }); test("the filter lists: switched on or off, how many rules, when updated — the year 1 is never", () => { const f = adg.parseFilters({ enabled: true, interval: 24, filters: [{ name: "AdGuard DNS filter", enabled: true, rules_count: 40000, last_updated: "2026-06-09T08:00:00+02:00" }, { name: "Old", enabled: false, last_updated: "0001-01-01T00:00:00Z" }, { name: "Never", enabled: true, last_updated: "0001-01-01T00:00:00Z" }, { enabled: true }] })!; assert.equal(f.enabled, true); assert.deepEqual(f.lists.map((l) => [l.name, l.enabled, l.rules, l.updated]), [["AdGuard DNS filter", true, 40000, "2026-06-09T06:00:00.000Z"], ["Old", false, null, null], ["Never", true, null, null]]); assert.equal(adg.parseFilters({ filters: null })!.lists.length, 0); assert.equal(adg.parseFilters("html"), null); }); describe("the adapter against an AdGuard Home that answers", () => { let http: Server; let url = ""; let dns: Awaited>; let dnsBehaviour: { rcode?: number; silent?: boolean } = {}; let authOk = true; let statsStatus = 200; let running = true; const seen: { method: string | undefined; path: string; auth: string | undefined }[] = []; before(async () => { dns = await fakeDns(() => dnsBehaviour); http = createHttpServer((req, res) => { const u = new URL(req.url ?? "", "http://x"); seen.push({ method: req.method, path: u.pathname, auth: req.headers.authorization }); res.setHeader("content-type", "application/json"); if (!authOk) { res.statusCode = 401; return void res.end("{}"); } if (u.pathname === "/control/status") return void res.end(JSON.stringify({ version: "v0.107.52", running, protection_enabled: true, dns_port: dns.port, http_port: 3000, dns_addresses: ["127.0.0.1"] })); if (u.pathname === "/control/stats") { if (statsStatus !== 200) { res.statusCode = statsStatus; return void res.end("{}"); } return void res.end(JSON.stringify({ time_units: "hours", num_dns_queries: 100, num_blocked_filtering: 20, avg_processing_time: 0.005, dns_queries: [1, 2], top_blocked_domains: [], top_clients: [], top_queried_domains: [] })); } if (u.pathname === "/control/filtering/status") return void res.end(JSON.stringify({ enabled: true, filters: [{ name: "L", enabled: true, rules_count: 10, last_updated: "2026-06-09T08:00:00Z" }] })); res.statusCode = 404; res.end("{}"); }); await new Promise((r) => http.listen(0, "127.0.0.1", r)); url = `http://127.0.0.1:${(http.address() as AddressInfo).port}`; }); after(() => { http.close(); dns.close(); }); test("status, statistics, lists and the DNS question — the sign-in as Basic, and only ever GET", async () => { seen.length = 0; const r = await adg.createAdguardAdapter({ url, username: "admin", password: "secret", probeDomain: "probe.example" }).read(); assert.deepEqual([r.version, r.running, r.protectionEnabled, r.dnsPort === dns.port, r.stats?.queries, r.filters?.lists.length], ["v0.107.52", true, true, true, 100, 1]); assert.deepEqual([r.probe.ok, r.probe.name, r.probe.host], [true, "probe.example", "127.0.0.1"]); assert.equal(dns.questions.at(-1), "probe.example"); assert.ok(seen.every((s) => s.method === "GET" && s.auth === `Basic ${Buffer.from("admin:secret").toString("base64")}`)); }); test("no sign-in sends none; a part that cannot be read is null, not empty; an unanswered DNS question is in the answer, not a failed read", async () => { seen.length = 0; statsStatus = 500; dnsBehaviour = { silent: true }; const r = await adg.createAdguardAdapter({ url }).read(); assert.deepEqual([r.stats, r.filters?.lists.length, r.probe.ok, seen.every((s) => s.auth === undefined)], [null, 1, false, true]); statsStatus = 200; dnsBehaviour = {}; }); test("a refused sign-in and an address that is not AdGuard Home are said", async () => { authOk = false; assert.match((await adg.createAdguardAdapter({ url, username: "a", password: "b" }).ping()).error ?? "", /did not accept the sign-in \(HTTP 401\)/); authOk = true; assert.equal((await adg.createAdguardAdapter({ url }).ping()).ok, true); await assert.rejects(() => adg.createAdguardAdapter({ url: `${url}/nowhere` }).read(), /no API at this address/); assert.equal((await adg.createAdguardAdapter({ url: "http://127.0.0.1:1" }).ping()).ok, false); }); test("the DNS server is asked at the host of the address, or the one named", async () => { const r = await adg.createAdguardAdapter({ url: url.replace("127.0.0.1", "localhost"), dnsHost: "127.0.0.1" }).read(); assert.equal(r.probe.host, "127.0.0.1"); }); }); }); describe("what AdGuard Home is carried and judged by", () => { const T0 = new Date("2026-06-10T10:00:00Z"); const read = (extra: Partial = {}) => ({ version: "v0.107", running: true, protectionEnabled: true, dnsPort: 53, stats: null, filters: null, probe: { ok: true, ms: 4, error: null, name: "example.com", host: "10.0.0.2" }, ...extra }); const stored = (extra: Partial = {}) => ({ version: "v0.107", running: true, protectionEnabled: true, protectionOffSince: null, dnsPort: 53, stats: null, filters: null, probe: { ok: true, ms: 4, error: null, name: "example.com", host: "10.0.0.2" }, probeFailSince: null, updatedAt: T0.toISOString(), ...extra }); const NOW = T0.getTime(); const keys = (list: ReturnType) => list.map((p) => `${p.key}=${p.severity}`); test("since when: a thing first seen wrong starts its clock now, keeps it while it stays wrong, and loses it when it is right", () => { const bad = read({ protectionEnabled: false, probe: { ok: false, ms: null, error: "no answer in 3 s", name: "example.com", host: "x" } }); const first = ag.carrySince(bad, null, T0); assert.deepEqual(first, { protectionOffSince: T0.toISOString(), probeFailSince: T0.toISOString() }); const later = ag.carrySince(bad, first, new Date(NOW + 5 * MIN)); assert.deepEqual(later, first); assert.deepEqual(ag.carrySince(read(), later, new Date(NOW + 9 * MIN)), { protectionOffSince: null, probeFailSince: null }); }); test("a DNS server that is not running is critical, and says so once (the test question is not raised beside it)", () => { const p = ag.adguardProblems(1, "Home DNS", stored({ running: false, probe: { ok: false, ms: null, error: "no answer in 3 s", name: "example.com", host: "x" }, probeFailSince: iso(NOW - HOUR) }), NOW); assert.deepEqual(keys(p), ["adguard-stopped:1=critical"]); assert.match(p[0].message, /Home DNS: AdGuard Home says its DNS server is not running/); }); test("a DNS question unanswered for ten minutes is critical; for less it is one lost packet", () => { const failing = (ago: number) => stored({ probe: { ok: false, ms: null, error: "no answer in 3 s", name: "example.com", host: "10.0.0.2" }, probeFailSince: iso(NOW - ago) }); assert.deepEqual(ag.adguardProblems(1, "D", failing(4 * MIN), NOW), []); const p = ag.adguardProblems(1, "D", failing(12 * MIN), NOW); assert.deepEqual(keys(p), ["adguard-dns:1=critical"]); assert.equal(p[0].message, "D: DNS has not answered a question about example.com for 12 minutes (asked 10.0.0.2:53) — no answer in 3 s"); assert.deepEqual(ag.adguardProblems(1, "D", stored(), NOW), []); }); test("protection off for an hour is a warning; for less it is a pause", () => { assert.deepEqual(ag.adguardProblems(1, "D", stored({ protectionEnabled: false, protectionOffSince: iso(NOW - 30 * MIN) }), NOW), []); const p = ag.adguardProblems(1, "D", stored({ protectionEnabled: false, protectionOffSince: iso(NOW - 3 * HOUR) }), NOW); assert.deepEqual(keys(p), ["adguard-protection:1=warning"]); assert.match(p[0].message, /protection has been switched off for 3 hours — nothing is being blocked/); }); test("filter lists switched on and not updated for a week (or never) are one warning; lists switched off, or filtering off, are not judged", () => { const list = (name: string, enabled: boolean, ageDays: number | null) => ({ name, enabled, rules: 1, updated: ageDays === null ? null : iso(NOW - ageDays * DAY) }); const lists = [list("fresh", true, 1), list("old", true, 9), list("never", true, null), list("off", false, 30)]; assert.deepEqual(ag.staleLists({ enabled: true, lists }, NOW).map((l) => l.name), ["old", "never"]); assert.deepEqual(ag.staleLists({ enabled: false, lists }, NOW), []); assert.deepEqual(ag.staleLists(null, NOW), []); const p = ag.adguardProblems(1, "D", stored({ filters: { enabled: true, lists } }), NOW); assert.deepEqual(keys(p), ["adguard-filters:1=warning"]); assert.equal(p[0].message, "D: 2 filter lists have not been updated for over a week — old, never"); }); }); describe("what AdGuard Home has stored", () => { let id = 0; before(async () => { id = (await db.insert(integrations).values({ type: "adguard", name: "DNS", baseUrl: "http://adguard", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id; }); const row = async () => (await db.select().from(adguardState).where(eq(adguardState.integrationId, id)))[0]; after(async () => { await ag.forgetAdguard(id); await db.delete(integrations).where(eq(integrations.id, id)); }); const bad = { version: "v1", running: true, protectionEnabled: false, dnsPort: 53, stats: null, filters: null, probe: { ok: false, ms: null, error: "no answer in 3 s", name: "example.com", host: "h" } }; test("the state is replaced at each read, carrying the clocks between them", async () => { const t0 = new Date(); await ag.storeRead(id, bad, t0); await ag.storeRead(id, bad, new Date(t0.getTime() + 2 * MIN)); const s = ag.storedOf(await row()); assert.deepEqual([s.protectionEnabled, s.protectionOffSince, s.probeFailSince, s.probe?.ok], [false, t0.toISOString(), t0.toISOString(), false]); await ag.storeRead(id, { ...bad, protectionEnabled: true, probe: { ...bad.probe, ok: true, ms: 3, error: null } }, new Date(t0.getTime() + 4 * MIN)); const healed = ag.storedOf(await row()); assert.deepEqual([healed.protectionOffSince, healed.probeFailSince], [null, null]); }); test("a read that failed is said, and what was stored is left alone", async () => { const ok = await ag.syncAdguard(id, new Date(), async () => ({ ...bad, version: "kept" })); assert.equal(ok.ok, true); const failed = await ag.syncAdguard(id, new Date(), async () => { throw new Error("connection refused"); }); assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]); assert.equal(failed.at, ok.at); assert.equal(ag.storedOf(await row()).version, "kept"); assert.equal(ag.isAdguardFresh((await ag.adguardSyncStatuses())[id], Date.now() + 11 * MIN), false, "ten minutes without a read is not believed"); }); }); describe("what Grafana's API says", () => { const apiRule = (name: string, extra: Record = {}) => ({ name, uid: `uid-${name}`, state: "inactive", health: "ok", labels: {}, annotations: {}, alerts: [], type: "alerting", ...extra }); const raw = (rules: unknown[]) => ({ status: "success", data: { groups: [{ name: "node", file: "Infra", rules }] } }); test("every rule is counted; only those firing, pending or in error are kept, with where they are, how serious and since when", () => { const r = gfa.parseRules( raw([ apiRule("quiet"), apiRule("loud", { state: "firing", labels: { severity: "Critical" }, annotations: { summary: "Disk is full" }, alerts: [{ activeAt: "2026-06-10T08:00:00Z" }, { activeAt: "2026-06-10T07:00:00Z" }] }), apiRule("soon", { state: "pending" }), apiRule("broken", { health: "error", lastError: "bad query" }), apiRule("nodata", { health: "nodata" }), { state: "firing" }, ]), ); assert.equal(r.total, 5); assert.deepEqual(r.items.map((i) => [i.name, i.state, i.health, i.severity, i.folder, i.group]), [["loud", "firing", "ok", "critical", "Infra", "node"], ["soon", "pending", "ok", null, "Infra", "node"], ["broken", "inactive", "error", null, "Infra", "node"]]); assert.deepEqual([r.items[0].summary, r.items[0].activeAt, r.items[2].lastError], ["Disk is full", "2026-06-10T07:00:00.000Z", "bad query"]); assert.deepEqual(gfa.parseRules(null), { total: 0, items: [] }); }); test("data sources: uid, name, type, default; health is OK, ERROR or not tested", () => { assert.deepEqual(gfa.parseDatasources([{ uid: "p1", name: "Prom", type: "prometheus", isDefault: true }, { name: "no uid" }, "junk"]), [{ uid: "p1", name: "Prom", type: "prometheus", isDefault: true }]); assert.deepEqual(gfa.parseDatasourceHealth({ status: "OK", message: "Data source is working" }), { ok: true, message: "Data source is working" }); assert.deepEqual(gfa.parseDatasourceHealth({ status: "ERROR", message: "connection refused" }), { ok: false, message: "connection refused" }); assert.deepEqual(gfa.parseDatasourceHealth({ message: "not found" }), { ok: null, message: null }); }); describe("the adapter against a Grafana that answers", () => { let http: Server; let url = ""; let tokenOk = true; let rulesStatus = 200; let listStatus = 200; const seen: { method: string | undefined; path: string; auth: string | undefined }[] = []; before(async () => { http = createHttpServer((req, res) => { const u = new URL(req.url ?? "", "http://x"); seen.push({ method: req.method, path: u.pathname, auth: req.headers.authorization }); res.setHeader("content-type", "application/json"); const send = (code: number, body: unknown) => { res.statusCode = code; res.end(JSON.stringify(body)); }; if (!tokenOk) return send(401, { message: "invalid token" }); if (u.pathname === "/api/health") return send(200, { database: "ok", version: "11.2.0" }); if (u.pathname === "/api/org") return send(200, { id: 1 }); if (u.pathname === "/api/prometheus/grafana/api/v1/rules") return rulesStatus === 200 ? send(200, raw([apiRule("loud", { state: "firing" }), apiRule("quiet")])) : send(rulesStatus, {}); if (u.pathname === "/api/datasources") return listStatus === 200 ? send(200, [{ uid: "p1", name: "Prom", type: "prometheus" }, { uid: "l1", name: "Loki", type: "loki" }, { uid: "t1", name: "Tempo", type: "tempo" }]) : send(listStatus, {}); if (u.pathname === "/api/datasources/uid/p1/health") return send(200, { status: "OK", message: "ok" }); if (u.pathname === "/api/datasources/uid/l1/health") return send(200, { status: "ERROR", message: "connection refused" }); if (u.pathname === "/api/datasources/uid/t1/health") return send(404, { message: "not found" }); send(404, {}); }); await new Promise((r) => http.listen(0, "127.0.0.1", r)); url = `http://127.0.0.1:${(http.address() as AddressInfo).port}`; }); after(() => http.close()); test("version, rules, data sources and each one's health — with the token as a Bearer, and only ever GET", async () => { seen.length = 0; const r = await gfa.createGrafanaAdapter({ url, token: "glsa_x" }).read(); assert.deepEqual([r.version, r.rules?.total, r.rules?.items.map((i) => i.name)], ["11.2.0", 2, ["loud"]]); assert.deepEqual(r.datasources?.map((d) => [d.name, d.ok, d.message]), [["Prom", true, "ok"], ["Loki", false, "connection refused"], ["Tempo", null, null]]); assert.ok(seen.every((s) => s.method === "GET" && s.auth === "Bearer glsa_x")); }); test("a part that cannot be read is null, not empty; when neither can, the read fails and says why", async () => { rulesStatus = 403; const r = await gfa.createGrafanaAdapter({ url, token: "t" }).read(); assert.deepEqual([r.rules, r.datasources?.length], [null, 3]); listStatus = 403; await assert.rejects(() => gfa.createGrafanaAdapter({ url, token: "t" }).read(), /neither the rules nor the data sources/); rulesStatus = 200; listStatus = 200; }); test("a token that is refused and an address that is not Grafana are said", async () => { tokenOk = false; const ping = await gfa.createGrafanaAdapter({ url, token: "bad" }).ping(); assert.equal(ping.ok, false); assert.match(ping.error ?? "", /did not accept the token \(HTTP 401\)/); tokenOk = true; assert.equal((await gfa.createGrafanaAdapter({ url, token: "t" }).ping()).ok, true); await assert.rejects(() => gfa.createGrafanaAdapter({ url: `${url}/nowhere`, token: "t" }).read(), /no API at this address/); assert.equal((await gfa.createGrafanaAdapter({ url: "http://127.0.0.1:1", token: "t" }).ping()).ok, false); }); }); }); describe("what is wrong with Grafana", () => { const NOW = Date.parse("2026-06-10T12:00:00Z"); const rule = (name: string, extra: Partial = {}) => ({ name, uid: `u-${name}`, group: "node", folder: "Infra", state: "inactive", health: "ok", lastError: null, severity: null, summary: null, activeAt: null, ...extra }); const state = (rules: ReturnType[] | null, datasources: import("../src/integrations/grafana/adapter.js").GrafanaDatasource[] | null = []) => ({ version: "11", rules: rules && { total: rules.length + 5, items: rules }, datasources, updatedAt: iso(NOW) }); const keys = (list: ReturnType) => list.map((p) => `${p.key}=${p.severity}`); test("a firing rule is a warning — critical when labelled critical or page — and says where, since when and why; a pending one is only shown", () => { const p = gs.grafanaProblems(1, "Graf", state([rule("disk", { state: "firing", severity: "critical", summary: "Disk is full", activeAt: iso(NOW - 3 * HOUR) }), rule("cpu", { state: "firing", severity: "warning" }), rule("soon", { state: "pending" }), rule("pager", { state: "firing", severity: "page" })]), NOW); assert.deepEqual(keys(p), ["grafana-firing:1:u-cpu=warning", "grafana-firing:1:u-disk=critical", "grafana-firing:1:u-pager=critical"]); assert.equal(p.find((x) => x.key.endsWith("u-disk"))!.message, 'Graf: the rule "disk" is firing (Infra / node) — for 3 hours — Disk is full'); }); test("a rule Grafana cannot evaluate is a warning with the reason; a rule that is firing and broken is both", () => { const p = gs.grafanaProblems(1, "G", state([rule("broken", { health: "error", lastError: "bad query" }), rule("both", { state: "firing", health: "error" })]), NOW); assert.deepEqual(keys(p).sort(), ["grafana-firing:1:u-both=warning", "grafana-rule-error:1:u-both=warning", "grafana-rule-error:1:u-broken=warning"]); assert.match(p.find((x) => x.key.endsWith("u-broken"))!.message, /cannot evaluate the rule "broken" \(Infra \/ node\) — bad query/); }); test("a data source that fails its check is a warning; one that works or was not tested is not", () => { const ds = (uid: string, ok: boolean | null) => ({ uid, name: uid.toUpperCase(), type: "loki", isDefault: false, ok, message: ok === false ? "connection refused" : null }); const p = gs.grafanaProblems(1, "G", state([], [ds("a", true), ds("b", false), ds("c", null)]), NOW); assert.deepEqual(keys(p), ["grafana-ds:1:b=warning"]); assert.equal(p[0].message, "G: the data source B (loki) fails Grafana's health check — connection refused"); }); test("lists that could not be read say nothing; a rule without a uid is keyed by its group and name", () => { assert.deepEqual(gs.grafanaProblems(1, "G", state(null, null), NOW), []); const p = gs.grafanaProblems(1, "G", state([rule("No Uid!", { uid: null, state: "firing" })]), NOW); assert.equal(p[0].key, "grafana-firing:1:node-no-uid"); }); }); describe("the health pass, the Alerts page, the pages' routes and /metrics", () => { let agId = 0; let gfId = 0; let app: Server; let base = ""; const fresh = async () => { await setInternalFlag("adguardSync", JSON.stringify({ [agId]: { at: new Date().toISOString(), ok: true, error: null } })); await setInternalFlag("grafanaSync", JSON.stringify({ [gfId]: { at: new Date().toISOString(), ok: true, error: null } })); }; const stale = async () => { await setInternalFlag("adguardSync", JSON.stringify({ [agId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } })); await setInternalFlag("grafanaSync", JSON.stringify({ [gfId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } })); }; before(async () => { await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]); agId = (await db.insert(integrations).values({ type: "adguard", name: "Home DNS", baseUrl: "http://adguard", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id; gfId = (await db.insert(integrations).values({ type: "grafana", name: "Dashboards", baseUrl: "http://grafana", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id; const t0 = new Date(Date.now() - 3 * HOUR); const down = { version: "v0.107", running: true, protectionEnabled: false, dnsPort: 53, stats: { queries: 1000, blocked: 250, blockedPercent: 25, avgMs: 8.5, safeBrowsing: 0, parental: 0, safeSearch: 0, periodHours: 24, topBlocked: [{ name: "ads.example", count: 99 }], topClients: [], topQueried: [] }, filters: { enabled: true, lists: [{ name: "Old list", enabled: true, rules: 5, updated: iso(Date.now() - 10 * DAY) }] }, probe: { ok: false, ms: null, error: "no answer in 3 s", name: "example.com", host: "10.0.0.2" } }; await ag.storeRead(agId, down, t0); await ag.storeRead(agId, down, new Date()); await gs.storeRead(gfId, { version: "11.2", rules: { total: 12, items: [{ name: "Disk full", uid: "d1", group: "node", folder: "Infra", state: "firing", health: "ok", lastError: null, severity: "critical", summary: "Disk is full", activeAt: iso(Date.now() - HOUR) }] }, datasources: [{ uid: "l1", name: "Loki", type: "loki", isDefault: false, ok: false, message: "connection refused" }, { uid: "p1", name: "Prom", type: "prometheus", isDefault: true, ok: true, message: null }] }, new Date()); await fresh(); const a = express(); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: "viewer" } }; next(); }); a.use("/api/adguard", adguardRouter); a.use("/api/grafana", grafanaRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("the health pass judges both from what is stored, and holds one whose last read is old; maintenance on the integration silences each", async () => { let { snapshot, held } = await collectSnapshot(); assert.deepEqual([snapshot.adguard?.map((x) => x.integrationId), snapshot.grafana?.map((x) => x.integrationId)], [[agId], [gfId]]); const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true }); const mine = list.filter((x) => /^(adguard|grafana)-/.test(x.key)).map((x) => `${x.key}=${x.severity}`).sort(); assert.deepEqual(mine, [`adguard-dns:${agId}=critical`, `adguard-filters:${agId}=warning`, `adguard-protection:${agId}=warning`, `grafana-ds:${gfId}:l1=warning`, `grafana-firing:${gfId}:d1=critical`].sort()); assert.equal(list.find((x) => x.key === `adguard-dns:${agId}`)!.source, `adguard:${agId}`); for (const k of [`adguard-dns:${agId}`, `adguard-stopped:${agId}`, `adguard-protection:${agId}`, `adguard-filters:${agId}`, `grafana-firing:${gfId}:x`, `grafana-rule-error:${gfId}:x`, `grafana-ds:${gfId}:x`]) { assert.equal(subjectOfConditionKey(k), `integration:${/^adguard/.test(k) ? agId : gfId}`, k); } await stale(); ({ snapshot, held } = await collectSnapshot()); assert.deepEqual([snapshot.adguard ?? [], snapshot.grafana ?? [], held.has(`adguard:${agId}`), held.has(`grafana:${gfId}`)], [[], [], true, true]); await fresh(); }); test("the Alerts page says where to look: DNS under Services, a Grafana rule under Monitoring, a failing data source under Integrations", async () => { invalidateAlertsCache(); const report = await getAlerts(true); const find = (key: string) => report.alerts.find((x) => x.id.endsWith(key))!; assert.deepEqual([find(`adguard-dns:${agId}`).category, find(`adguard-dns:${agId}`).source, find(`adguard-dns:${agId}`).link, find(`adguard-protection:${agId}`).category], ["services", "AdGuard Home", "/adguard", "monitoring"]); assert.deepEqual([find(`grafana-firing:${gfId}:d1`).category, find(`grafana-firing:${gfId}:d1`).link, find(`grafana-ds:${gfId}:l1`).category, find(`grafana-firing:${gfId}:d1`).target?.type], ["monitoring", "/grafana", "integration", "integration"]); }); test("the pages: AdGuard Home with its DNS question, statistics, lists and what is wrong; Grafana with its rules and data sources; an old reading says nothing is wrong", async () => { const a = (await (await fetch(`${base}/adguard`)).json()) as { instances: { id: number; fresh: boolean; problems: unknown[]; state: { probe: { ok: boolean }; staleLists: string[]; stats: { queries: number }; protectionOffSince: string | null } }[] }; const adg0 = a.instances.find((x) => x.id === agId)!; assert.deepEqual([adg0.fresh, adg0.problems.length, adg0.state.probe.ok, adg0.state.staleLists, adg0.state.stats.queries, adg0.state.protectionOffSince !== null], [true, 3, false, ["Old list"], 1000, true]); const g = (await (await fetch(`${base}/grafana`)).json()) as { instances: { id: number; problems: unknown[]; state: { rules: { total: number; items: { name: string }[] }; datasources: { name: string; ok: boolean | null }[] } }[] }; const gf0 = g.instances.find((x) => x.id === gfId)!; assert.deepEqual([gf0.problems.length, gf0.state.rules.total, gf0.state.rules.items[0].name, gf0.state.datasources.map((d) => [d.name, d.ok])], [2, 12, "Disk full", [["Loki", false], ["Prom", true]]]); await stale(); const old = (await (await fetch(`${base}/adguard`)).json()) as typeof a; assert.deepEqual(old.instances.find((x) => x.id === agId)!.problems, []); await fresh(); }); test("/metrics has DNS up and how fast, protection, the queries, Grafana's rules firing and failing and the data sources failing — only while the reading is recent", async () => { let text = await buildMetrics(new Date()); assert.match(text, /homelab_adguard_running\{name="Home DNS"\} 1/); assert.match(text, /homelab_adguard_protection_enabled\{name="Home DNS"\} 0/); assert.match(text, /homelab_adguard_dns_up\{name="Home DNS"\} 0/); assert.doesNotMatch(text, /homelab_adguard_dns_probe_seconds/, "no answer, no time"); assert.match(text, /homelab_adguard_queries\{name="Home DNS"\} 1000/); assert.match(text, /homelab_adguard_avg_processing_seconds\{name="Home DNS"\} 0\.0085/); assert.match(text, /homelab_grafana_rules\{name="Dashboards"\} 12/); assert.match(text, /homelab_grafana_rules_firing\{name="Dashboards"\} 1/); assert.match(text, /homelab_grafana_datasources_failing\{name="Dashboards"\} 1/); await stale(); text = await buildMetrics(new Date()); assert.doesNotMatch(text, /homelab_(adguard|grafana)_/); await fresh(); }); test("taking the integrations away takes what was read with them", async () => { await ag.forgetAdguard(agId); await gs.forgetGrafana(gfId); assert.equal((await ag.adguardSyncStatuses())[agId], undefined); await db.delete(integrations).where(eq(integrations.id, agId)); await db.delete(integrations).where(eq(integrations.id, gfId)); assert.deepEqual(await db.select().from(adguardState).where(eq(adguardState.integrationId, agId)), []); assert.deepEqual(await db.select().from(grafanaState).where(eq(grafanaState.integrationId, gfId)), []); }); });