AdGuard Home, Grafana, Docker image age, foldable server page cards

New integration kinds adguard (status, statistics, filter lists and a real DNS question
over UDP; alerts for DNS down/not running, protection left off, stale lists) and grafana
(alert rules firing or failing, data-source health). Optional warning for running Docker
images built more than N days ago (off by default). A server's overview cards can be
folded and arranged, kept per user. Migrations 0075-0077; docs, metrics and tests included.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-27 22:13:03 +01:00
1 parent 3d9a2310bb
commit 551eb1597a
53 files changed
+27700 -33

No files matched your search

+4
View File
@@ -71,6 +71,8 @@ import { speedtestRouter } from "./routes/speedtest.js";
import { fragnetRouter } from "./routes/fragnet.js";
import { arrRouter } from "./routes/arr.js";
import { wallRouter } from "./routes/wall.js";
import { adguardRouter } from "./routes/adguard.js";
import { grafanaRouter } from "./routes/grafana.js";
import { costsRouter } from "./routes/costs.js";
import { nextcloudRouter } from "./routes/nextcloud.js";
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
@@ -220,6 +222,8 @@ export function createApp() {
app.use("/api/fragnet", fragnetRouter);
app.use("/api/arr", arrRouter);
app.use("/api/wall", wallRouter);
app.use("/api/adguard", adguardRouter);
app.use("/api/grafana", grafanaRouter);
app.use("/api/costs", costsRouter);
app.use("/api/nextcloud", nextcloudRouter);
app.use("/api/service-levels", serviceLevelsRouter);
+42
View File
@@ -26,6 +26,9 @@ export const users = sqliteTable("users", {
// How this person arranged the dashboard: a JSON object with the cards they hid ("hidden": ["Gitea"]) and the order of the others
// ("order": {"DNS": 0, "Secrets": 1}). Follows them to any browser; null is the default layout.
dashboardLayout: text("dashboard_layout"),
// How this person arranged a server's overview page: the cards they folded ("folded": ["admin-links"]) and the order of the blocks ("order": {"notes": 10}). The same for every server;
// follows them to any browser; null is the page as drawn.
serverPageLayout: text("server_page_layout"),
// The address of the picture the identity provider gave for this person (the OIDC "picture" claim), as of their last sign-in. Only an address:
// nothing is fetched from it unless Settings → Sign-in → Profile pictures allows it, and then only by this server, never by the browser.
pictureUrl: text("picture_url"),
@@ -567,6 +570,8 @@ export const integrationTypes = [
"sonarr",
"radarr",
"prowlarr",
"adguard",
"grafana",
] as const;
export type IntegrationType = (typeof integrationTypes)[number];
@@ -1206,6 +1211,8 @@ export const dockhandVulnerabilities = sqliteTable(
imagesScanned: integer("images_scanned"),
totalImages: integer("total_images"),
findings: text("findings").notNull().default("[]"), // JSON: the critical findings (cve, package, installedVersion, fixedVersion, imageName, containers), at most 100
images: text("images"), // JSON: the images the running containers of this host use, each with when it was built (image, created, containers); null: never read, or could not be
imagesAt: text("images_at"), // ISO: when `images` was last read
updatedAt: text("updated_at").notNull(), // ISO
},
(t) => [primaryKey({ columns: [t.integrationId, t.environmentId] })],
@@ -1322,6 +1329,41 @@ export const arrState = sqliteTable("arr_state", {
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── AdGuard Home ───────────────────────────────────────────────────────────
// How an AdGuard Home (an integration of type "adguard") is right now, one row each, replaced at every read: whether its DNS server runs and protects, the statistics, its filter lists and the result
// of a real DNS question put to it — plus since when protection has been off and the question unanswered, which only this app can know. A column that is null could not be read. Read by
// services/adguardSync.ts.
export const adguardState = sqliteTable("adguard_state", {
integrationId: integer("integration_id")
.primaryKey()
.references(() => integrations.id, { onDelete: "cascade" }),
version: text("version"),
running: integer("running", { mode: "boolean" }).notNull().default(true),
protectionEnabled: integer("protection_enabled", { mode: "boolean" }).notNull().default(true),
protectionOffSince: text("protection_off_since"), // ISO: the first read that showed protection off; null while it is on
dnsPort: integer("dns_port").notNull().default(53),
stats: text("stats"), // JSON: queries, blocked, average time, top names and clients; null: could not be read
filters: text("filters"), // JSON: { enabled, lists: [{ name, enabled, rules, updated }] }; null: could not be read
probe: text("probe"), // JSON: the DNS question — { ok, ms, error, name, host }
probeFailSince: text("probe_fail_since"), // ISO: the first read in which the question went unanswered; null while it is answered
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── Grafana ────────────────────────────────────────────────────────────────
// How a Grafana (an integration of type "grafana") is right now, one row each, replaced at every read: the alert rules that are firing, pending or cannot be evaluated (and how many there are), and
// its data sources with the result of Grafana's own health check of each. A column that is null could not be read. Read by services/grafanaSync.ts.
export const grafanaState = sqliteTable("grafana_state", {
integrationId: integer("integration_id")
.primaryKey()
.references(() => integrations.id, { onDelete: "cascade" }),
version: text("version"),
rules: text("rules"), // JSON { total, items }; null: could not be read
datasources: text("datasources"), // JSON; null: could not be listed
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── NetBox ─────────────────────────────────────────────────────────────────
// How a NetBox (an integration of type "netbox") is right now, one row each, replaced at every read: its version, how many background workers are running, the counts and the IP
+4
View File
@@ -42,6 +42,8 @@ import { initHetznerSync } from "./services/hetznerSync.js";
import { initSpeedtestSync } from "./services/speedtestSync.js";
import { initFragnetSync } from "./services/fragnetSync.js";
import { initArrSync } from "./services/arrSync.js";
import { initAdguardSync } from "./services/adguardSync.js";
import { initGrafanaSync } from "./services/grafanaSync.js";
import { backfillMetricRollups } from "./services/serverMetrics.js";
import { backfillEndpointHourly } from "./services/endpointChecks.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
@@ -105,6 +107,8 @@ initHetznerSync();
initSpeedtestSync();
initFragnetSync();
initArrSync();
initAdguardSync();
initGrafanaSync();
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
+189
View File
@@ -0,0 +1,189 @@
/**
* AdGuard Home adapter. Config: url (its web interface), username and password (its own sign-in — HTTP Basic; leave them empty if it has none), optionally dnsHost (where its DNS server is, when
* that is not the host of the web address — a reverse proxy in front of the interface, say), probeDomain (the name a test question is asked about; example.com unless told) and insecure.
*
* Reading only: whether it runs and protects (`GET /control/status`), what it has done in the statistics period (`GET /control/stats`: queries, how many blocked, how long an answer takes, the
* names blocked most and the clients asking most), its filter lists and when each was last updated (`GET /control/filtering/status`) — and one real DNS question put to its DNS server over UDP
* (see dnsProbe.ts), because the interface can be up while DNS is not. Nothing is changed: protection is never switched, no list refreshed.
*
* Endpoints and fields are from AdGuard Home's published OpenAPI file (openapi/openapi.yaml: ServerStatus, Stats, FilterStatus) and have not been checked against a live install here; every
* field is read defensively.
*/
import { requestJson } from "../httpJson.js";
import { withDiagLogging } from "../../services/diagLog.js";
import { probeDns, type ProbeResult } from "./dnsProbe.js";
export interface AdguardConfig {
url: string;
username?: string;
password?: string;
insecure?: boolean;
dnsHost?: string;
probeDomain?: string;
}
export const DEFAULT_PROBE_DOMAIN = "example.com";
export interface AdguardCounted {
name: string;
count: number;
}
export interface AdguardStats {
/** Over AdGuard Home's statistics period (a day unless it was set otherwise): see `periodHours`. */
queries: number;
blocked: number;
/** Blocked as a share of the queries, in percent; null with no queries. */
blockedPercent: number | null;
/** Average time to answer, in milliseconds; null when not said. */
avgMs: number | null;
/** Of the blocked ones, the ones made safe by Safe Browsing, parental control and safe search. */
safeBrowsing: number;
parental: number;
safeSearch: number;
periodHours: number | null;
topBlocked: AdguardCounted[];
topClients: AdguardCounted[];
topQueried: AdguardCounted[];
}
export interface AdguardFilterList {
name: string;
enabled: boolean;
rules: number | null;
/** ISO; null when it has never been updated. */
updated: string | null;
}
export interface AdguardRead {
version: string | null;
/** Whether its DNS server is running. */
running: boolean;
protectionEnabled: boolean;
dnsPort: number;
/** Null when the statistics could not be read. */
stats: AdguardStats | null;
/** Null when the filter lists could not be read. */
filters: { enabled: boolean; lists: AdguardFilterList[] } | null;
/** The real DNS question: whether it was answered, how fast, and about what. */
probe: ProbeResult & { name: string; host: string };
}
export interface AdguardAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
read(): Promise<AdguardRead>;
}
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
const arr = (v: unknown): unknown[] => (Array.isArray(v) ? v : []);
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
/** Pure: AdGuard's lists of counts — `[{ "example.com": 12 }, { "ads.example": 7 }]`, one name to a pair. */
export function parseCounted(raw: unknown, limit = 10): AdguardCounted[] {
const out: AdguardCounted[] = [];
for (const pair of arr(raw)) {
for (const [name, count] of Object.entries(obj(pair))) {
const n = num(count);
if (n !== null) out.push({ name, count: n });
}
}
return out.sort((a, b) => b.count - a.count).slice(0, limit);
}
/** Pure: `/control/stats`. */
export function parseStats(raw: unknown): AdguardStats | null {
const s = obj(raw);
const queries = num(s.num_dns_queries);
if (queries === null) return null;
const blocked = num(s.num_blocked_filtering) ?? 0;
const avg = num(s.avg_processing_time);
const units = str(s.time_units);
const slots = arr(s.dns_queries).length;
return {
queries,
blocked,
blockedPercent: queries > 0 ? Math.round((blocked / queries) * 1000) / 10 : null,
avgMs: avg === null ? null : Math.round(avg * 10000) / 10,
safeBrowsing: num(s.num_replaced_safebrowsing) ?? 0,
parental: num(s.num_replaced_parental) ?? 0,
safeSearch: num(s.num_replaced_safesearch) ?? 0,
periodHours: units === "hours" && slots > 0 ? slots : units === "days" && slots > 0 ? slots * 24 : null,
topBlocked: parseCounted(s.top_blocked_domains),
topClients: parseCounted(s.top_clients),
topQueried: parseCounted(s.top_queried_domains),
};
}
/** Pure: `/control/filtering/status` — the lists, and whether filtering is on at all. AdGuard writes "never updated" as the year 1. */
export function parseFilters(raw: unknown): { enabled: boolean; lists: AdguardFilterList[] } | null {
const f = obj(raw);
if (!("filters" in f) && !("enabled" in f)) return null;
const lists: AdguardFilterList[] = [];
for (const item of arr(f.filters)) {
const l = obj(item);
const name = str(l.name) ?? str(l.url);
if (!name) continue;
const updated = str(l.last_updated);
const t = updated === null ? NaN : Date.parse(updated);
lists.push({ name, enabled: l.enabled !== false, rules: num(l.rules_count), updated: Number.isFinite(t) && t > Date.UTC(2000, 0, 1) ? new Date(t).toISOString() : null });
}
return { enabled: f.enabled !== false, lists };
}
export function createAdguardAdapter(config: AdguardConfig): AdguardAdapter {
const raw = String(config.url ?? "").trim().replace(/\/+$/, "");
const base = raw;
const insecure = config.insecure === true;
const headers: Record<string, string> = { Accept: "application/json" };
if (config.username) headers.Authorization = `Basic ${Buffer.from(`${config.username}:${config.password ?? ""}`).toString("base64")}`;
async function get(path: string): Promise<unknown> {
const res = await requestJson(`${base}${path}`, { headers, insecure });
if (res.status === 401 || res.status === 403) throw new Error(`AdGuard Home did not accept the sign-in (HTTP ${res.status}) — check the user name and password`);
if (res.status === 404) throw new Error("AdGuard Home has no API at this address (HTTP 404) — check the address (its own, not a proxy's)");
if (res.status < 200 || res.status >= 300) throw new Error(`AdGuard Home error: HTTP ${res.status}`);
if (res.json === null) throw new Error("AdGuard Home answered with something that isn't JSON — is the address its own?");
return res.json;
}
const soft = async <T>(run: () => Promise<T>): Promise<T | null> => {
try {
return await run();
} catch {
return null;
}
};
function dnsHostOf(): string {
const given = str(config.dnsHost);
if (given) return given.replace(/^\[|\]$/g, "");
try {
return new URL(raw).hostname.replace(/^\[|\]$/g, "");
} catch {
return "";
}
}
async function read(): Promise<AdguardRead> {
// The status must work: it says it is there, whether it runs and whether it protects.
const status = obj(await get("/control/status"));
const dnsPort = num(status.dns_port) ?? 53;
const name = str(config.probeDomain) ?? DEFAULT_PROBE_DOMAIN;
const host = dnsHostOf();
const [stats, filters, probe] = await Promise.all([soft(async () => parseStats(await get("/control/stats"))), soft(async () => parseFilters(await get("/control/filtering/status"))), host ? probeDns(host, dnsPort, name) : Promise.resolve({ ok: false, ms: null, error: "no address to ask" } as ProbeResult)]);
return { version: str(status.version), running: status.running !== false, protectionEnabled: status.protection_enabled !== false, dnsPort, stats, filters, probe: { ...probe, name, host } };
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
const start = Date.now();
try {
await get("/control/status");
return { ok: true, latencyMs: Date.now() - start };
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) };
}
}
return withDiagLogging("adguard", { ping, read });
}
@@ -0,0 +1,83 @@
/**
* Asks a DNS server a plain question over UDP — "what is the address of example.com?" — and says whether it answered. That is the only honest test of "DNS is working": AdGuard Home's own API
* can be up while its DNS server is not (or its upstreams are not answering), and what a household notices is a name that does not resolve. Nothing is read from the answer but whether there is
* one and how it ended: an "address not found" (NXDOMAIN) is an answer too — the resolver works — while a server failure (SERVFAIL, REFUSED …) is not.
*/
import * as dgram from "node:dgram";
import * as net from "node:net";
import { randomInt } from "node:crypto";
export const PROBE_TIMEOUT_MS = 3000;
/** Pure: a standard recursive query for the address (A record) of a name. */
export function buildQuery(name: string, id: number): Buffer {
const labels = name.replace(/\.$/, "").split(".").filter((l) => l !== "");
if (labels.length === 0 || labels.some((l) => l.length > 63 || !/^[A-Za-z0-9_-]+$/.test(l))) throw new Error(`"${name}" is not a name to ask about`);
const header = Buffer.alloc(12);
header.writeUInt16BE(id & 0xffff, 0);
header.writeUInt16BE(0x0100, 2); // recursion desired
header.writeUInt16BE(1, 4); // one question
const parts = labels.map((l) => Buffer.concat([Buffer.from([l.length]), Buffer.from(l, "ascii")]));
const tail = Buffer.alloc(4);
tail.writeUInt16BE(1, 0); // A
tail.writeUInt16BE(1, 2); // IN
return Buffer.concat([header, ...parts, Buffer.from([0]), tail]);
}
const RCODES = ["no error", "format error", "server failure (SERVFAIL)", "no such name (NXDOMAIN)", "not implemented", "refused (REFUSED)"];
/** Pure: whether a packet is the answer to the query with this id, and how it ended; null when it is not an answer to it. */
export function parseResponse(packet: Buffer, id: number): { rcode: number; answers: number } | null {
if (packet.length < 12) return null;
if (packet.readUInt16BE(0) !== (id & 0xffff)) return null;
const flags = packet.readUInt16BE(2);
if ((flags & 0x8000) === 0) return null; // a query, not a response
return { rcode: flags & 0x000f, answers: packet.readUInt16BE(6) };
}
export interface ProbeResult {
ok: boolean;
/** How long the answer took, in milliseconds; null when there was none. */
ms: number | null;
error: string | null;
}
/** Whether a DNS server answers a question for a name. Never throws. */
export function probeDns(host: string, port: number, name: string, timeoutMs: number = PROBE_TIMEOUT_MS): Promise<ProbeResult> {
return new Promise((resolve) => {
let query: Buffer;
const id = randomInt(0, 0x10000);
try {
query = buildQuery(name, id);
} catch (err) {
return resolve({ ok: false, ms: null, error: err instanceof Error ? err.message : String(err) });
}
const socket = dgram.createSocket(net.isIPv6(host) ? "udp6" : "udp4");
const started = Date.now();
let done = false;
const finish = (result: ProbeResult) => {
if (done) return;
done = true;
clearTimeout(timer);
try {
socket.close();
} catch {
// already closed
}
resolve(result);
};
const timer = setTimeout(() => finish({ ok: false, ms: null, error: `no answer in ${Math.round(timeoutMs / 1000)} s` }), timeoutMs);
socket.on("error", (err) => finish({ ok: false, ms: null, error: err.message }));
socket.on("message", (packet) => {
const r = parseResponse(packet, id);
if (!r) return; // not ours: keep waiting
const ms = Date.now() - started;
// An answer, or "no such name", means the resolver works; any other ending means it is up but cannot resolve.
if (r.rcode === 0 || r.rcode === 3) finish({ ok: true, ms, error: null });
else finish({ ok: false, ms, error: `answered, but with ${RCODES[r.rcode] ?? `error ${r.rcode}`}` });
});
socket.send(query, port, host, (err) => {
if (err) finish({ ok: false, ms: null, error: err.message });
});
});
}
+87 -1
View File
@@ -122,6 +122,23 @@ export interface DockhandEnvironmentVulns {
critical: DockhandVulnFinding[];
}
/** An image a running container uses, and when that image was built (Docker's own "Created"). */
export interface DockhandRunningImage {
/** As the container names it ("nginx:1.25"). */
image: string;
/** ISO; null when the image could not be found in the list (an untagged one, a name that was typed differently). */
created: string | null;
/** The running containers that use it. */
containers: string[];
}
export interface DockhandEnvironmentImages {
environmentId: number;
environmentName: string;
/** Null when the images or the containers of this environment could not be read — which is not the same as none. */
images: DockhandRunningImage[] | null;
}
export interface DockhandVulnerabilityRead {
environments: DockhandEnvironmentVulns[];
}
@@ -141,6 +158,56 @@ export function parseVulnSummary(raw: unknown): DockhandVulnSummary | null {
return { critical, high: asCount(s.high) ?? 0, medium: asCount(s.medium) ?? 0, low: asCount(s.low) ?? 0, imagesScanned: asCount(s.imagesScanned), totalImages: asCount(s.totalImages) };
}
/** Pure: when an image was built, from Docker's "Created" — seconds since 1970 (what Docker's list gives), milliseconds, or a date; null for anything else, and for Docker's "never" (0). */
export function parseImageCreated(v: unknown): string | null {
if (typeof v === "number" && Number.isFinite(v) && v > 0) {
const ms = v < 1e11 ? v * 1000 : v;
return new Date(ms).toISOString();
}
if (typeof v === "string" && v.trim() !== "") {
const t = Date.parse(v);
if (Number.isFinite(t) && t > 0) return new Date(t).toISOString();
}
return null;
}
/** Pure: the images of /api/images — Docker's own list: `Id`, `RepoTags` and `Created` (a lower-case spelling is read as well). */
export function parseImageList(raw: unknown): { id: string; tags: string[]; created: string | null }[] {
const out: { id: string; tags: string[]; created: string | null }[] = [];
for (const item of Array.isArray(raw) ? raw : []) {
const i = (item !== null && typeof item === "object" ? item : {}) as Record<string, unknown>;
const id = asStr(i.Id) ?? asStr(i.id);
if (!id) continue;
const tags = (Array.isArray(i.RepoTags) ? i.RepoTags : Array.isArray(i.repoTags) ? i.repoTags : []).map(asStr).filter((t): t is string => t !== null && t !== "<none>:<none>");
out.push({ id, tags, created: parseImageCreated(i.Created ?? i.created) });
}
return out;
}
/** An image name as Docker lists it: without the registry Docker Hub is by default, and with a tag. */
const canonicalImage = (name: string): string => {
const bare = name.replace(/^(?:docker\.io\/)?(?:library\/)?/, "");
const last = bare.split("/").pop() ?? bare;
return last.includes(":") || bare.includes("@") ? bare : `${bare}:latest`;
};
/** Pure: the images the running containers use, each with when it was built — matched by the name the container gives (tag), or by id when it gives one. */
export function matchRunningImages(containers: { name: string; image: string; state: string }[], images: { id: string; tags: string[]; created: string | null }[]): DockhandRunningImage[] {
const byTag = new Map<string, string | null>();
for (const i of images) for (const t of i.tags) byTag.set(canonicalImage(t), i.created);
const byId = new Map(images.map((i) => [i.id.replace(/^sha256:/, ""), i.created]));
const out = new Map<string, DockhandRunningImage>();
for (const c of containers) {
if (c.state !== "running" || !c.image) continue;
const sha = c.image.replace(/^sha256:/, "");
const created = byTag.get(canonicalImage(c.image)) ?? (/^[0-9a-f]{12,64}$/.test(sha) ? ([...byId.entries()].find(([id]) => id.startsWith(sha))?.[1] ?? null) : null);
const entry = out.get(c.image) ?? { image: c.image, created: created ?? null, containers: [] };
entry.containers.push(c.name);
out.set(c.image, entry);
}
return [...out.values()].sort((a, b) => a.image.localeCompare(b.image));
}
/** Pure: the findings of one page of /api/vulnerabilities — each `{ cve, package, severity, installedVersion, fixedVersion, imageName, containers: [{ id, name }] }`. */
export function parseVulnFindings(raw: unknown): DockhandVulnFinding[] {
const list = asObj(raw).findings;
@@ -160,6 +227,8 @@ export interface DockhandAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
/** What Dockhand's own stored scans say, per environment. Reads only: no scan is started. */
readVulnerabilities(): Promise<DockhandVulnerabilityRead>;
/** For each Docker host, the images its running containers use and when each was built (GET /api/images and /api/containers). Reads only. */
readImages(): Promise<{ environments: DockhandEnvironmentImages[] }>;
listContainers(): Promise<DockhandContainer[]>;
listContainersDetailed(): Promise<DockhandContainerRead>;
startContainer(environmentId: number, containerId: string): Promise<void>;
@@ -269,6 +338,23 @@ export function createDockhandAdapter(config: DockhandConfig): DockhandAdapter {
return { environments: perEnv };
}
async function readImages(): Promise<{ environments: DockhandEnvironmentImages[] }> {
const environments = await listEnvironments();
const perEnv = await Promise.all(
environments.map(async (env): Promise<DockhandEnvironmentImages> => {
const known = { environmentId: env.id, environmentName: env.name };
try {
const [list, containers] = await Promise.all([api("GET", `/api/images?env=${numericSegment(env.id, "environment ID")}`), api("GET", `/api/containers?env=${numericSegment(env.id, "environment ID")}&all=true`)]);
const running = (Array.isArray(containers) ? containers : []).map((c: any) => ({ name: String(c.name ?? c.id ?? ""), image: String(c.image ?? ""), state: String(c.state ?? "") }));
return { ...known, images: matchRunningImages(running, parseImageList(list)) };
} catch {
return { ...known, images: null };
}
}),
);
return { environments: perEnv };
}
async function listContainers(): Promise<DockhandContainer[]> {
return (await listContainersDetailed()).containers;
}
@@ -314,5 +400,5 @@ export function createDockhandAdapter(config: DockhandConfig): DockhandAdapter {
}
}
return withDiagLogging("dockhand", { ping, readVulnerabilities, listContainers, listContainersDetailed, startContainer, stopContainer, restartContainer, checkForUpdates });
return withDiagLogging("dockhand", { ping, readVulnerabilities, readImages, listContainers, listContainersDetailed, startContainer, stopContainer, restartContainer, checkForUpdates });
}
+13
View File
@@ -129,6 +129,19 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "Settings → General → Security → API Key" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
adguard: [
{ key: "url", label: "AdGuard Home URL", secret: false, placeholder: "http://adguard.example.lan:3000" },
{ key: "username", label: "User name (optional)", secret: false, optional: true, placeholder: "empty when AdGuard Home has no sign-in" },
{ key: "password", label: "Password (optional)", secret: true, optional: true, type: "password" },
{ key: "dnsHost", label: "DNS server address (optional)", secret: false, optional: true, placeholder: "the host of the URL — only if its DNS server is somewhere else" },
{ key: "probeDomain", label: "Name to test DNS with (optional)", secret: false, optional: true, placeholder: "example.com" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
grafana: [
{ key: "url", label: "Grafana URL", secret: false, placeholder: "http://grafana.example.lan:3000" },
{ key: "token", label: "Service account token", secret: true, type: "password", placeholder: "Administration → Service accounts → add a token (role Viewer)" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
fragnet: [
{ key: "token", label: "API key", secret: true, type: "password", placeholder: "ptlc_… — the game panel: profile → API Credentials (not an Application key, ptla_)" },
{ key: "url", label: "Panel address (optional)", secret: false, optional: true, placeholder: "https://panel.fragnet.net — only if yours is somewhere else" },
+198
View File
@@ -0,0 +1,198 @@
/**
* Grafana adapter. Config: url and token (a service account's token — Administration → Users and access → Service accounts → add a token; the **Viewer** role is enough for the rules, and to also
* test the data sources the account needs permission to read and query them) and optionally insecure.
*
* Reading only: the alert rules and what state each is in (`GET /api/prometheus/grafana/api/v1/rules`, the Prometheus-compatible view of Grafana-managed alerting: firing, pending, and whether it
* could be evaluated), and the data sources with the result of Grafana's own health check of each (`GET /api/datasources`, `GET /api/datasources/uid/{uid}/health`). Nothing is changed: no rule is
* silenced or paused, no data source edited.
*
* Endpoints and fields are from Grafana's HTTP API documentation and have not been checked against a live Grafana here; every field is read defensively. A data source whose type has no health
* check is "not tested" (null), which is not the same as healthy.
*/
import { requestJson } from "../httpJson.js";
import { withDiagLogging } from "../../services/diagLog.js";
export interface GrafanaConfig {
url: string;
token: string;
insecure?: boolean;
}
export interface GrafanaRule {
name: string;
uid: string | null;
group: string;
/** The folder it is in (Grafana's "file"). */
folder: string | null;
/** inactive, pending or firing. */
state: string;
/** ok, error or nodata. */
health: string;
lastError: string | null;
/** The rule's own `severity` label, lower case, when it has one. */
severity: string | null;
summary: string | null;
/** ISO: when the oldest of its alerts started. */
activeAt: string | null;
}
export interface GrafanaDatasource {
uid: string;
name: string;
type: string;
isDefault: boolean;
/** The result of Grafana's health check; null when the type has none or it could not be asked. */
ok: boolean | null;
message: string | null;
}
export interface GrafanaRead {
version: string | null;
/** The rules that are firing, pending or cannot be evaluated, and how many rules there are in all; null when the rules could not be read. */
rules: { total: number; items: GrafanaRule[] } | null;
/** Null when the data sources could not be listed. */
datasources: GrafanaDatasource[] | null;
}
export interface GrafanaAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
read(): Promise<GrafanaRead>;
}
const MAX_DATASOURCES = 40;
const MAX_ITEMS = 200;
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
const arr = (v: unknown): unknown[] => (Array.isArray(v) ? v : []);
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
const iso = (v: unknown): string | null => {
const s = str(v);
const t = s === null ? NaN : Date.parse(s);
// Grafana writes "no time" as the year 1.
return Number.isFinite(t) && t > Date.UTC(2000, 0, 1) ? new Date(t).toISOString() : null;
};
/** Pure: the rules of `/api/prometheus/grafana/api/v1/rules` — every rule counted, those that are not quietly fine kept. */
export function parseRules(raw: unknown): { total: number; items: GrafanaRule[] } {
let total = 0;
const items: GrafanaRule[] = [];
for (const g of arr(obj(obj(raw).data).groups)) {
const group = obj(g);
const groupName = str(group.name) ?? "";
for (const r of arr(group.rules)) {
const rule = obj(r);
const name = str(rule.name);
if (!name) continue;
total++;
const state = (str(rule.state) ?? "inactive").toLowerCase();
const health = (str(rule.health) ?? "ok").toLowerCase();
if (state === "inactive" && health !== "error") continue;
if (state === "normal" && health !== "error") continue;
const labels = obj(rule.labels);
const annotations = obj(rule.annotations);
const activeAt = arr(rule.alerts)
.map((a) => iso(obj(a).activeAt))
.filter((t): t is string => t !== null)
.sort()[0] ?? null;
items.push({
name,
uid: str(rule.uid),
group: groupName,
folder: str(group.file),
state: state === "alerting" ? "firing" : state,
health,
lastError: str(rule.lastError),
severity: str(labels.severity)?.toLowerCase() ?? null,
summary: str(annotations.summary) ?? str(annotations.description),
activeAt,
});
}
}
return { total, items: items.slice(0, MAX_ITEMS) };
}
/** Pure: the data sources of `/api/datasources` (health not yet asked). */
export function parseDatasources(raw: unknown): Omit<GrafanaDatasource, "ok" | "message">[] {
const out: Omit<GrafanaDatasource, "ok" | "message">[] = [];
for (const d of arr(raw)) {
const o = obj(d);
const uid = str(o.uid);
const name = str(o.name);
if (!uid || !name) continue;
out.push({ uid, name, type: str(o.type) ?? "unknown", isDefault: o.isDefault === true });
}
return out;
}
/** Pure: Grafana's health answer for one data source — `{ status: "OK" | "ERROR", message }`; anything else is "not tested". */
export function parseDatasourceHealth(raw: unknown): { ok: boolean | null; message: string | null } {
const h = obj(raw);
const status = str(h.status)?.toUpperCase();
if (status === "OK") return { ok: true, message: str(h.message) };
if (status === "ERROR") return { ok: false, message: str(h.message) };
return { ok: null, message: null };
}
export function createGrafanaAdapter(config: GrafanaConfig): GrafanaAdapter {
const base = String(config.url ?? "").trim().replace(/\/+$/, "");
const insecure = config.insecure === true;
const headers = { Authorization: `Bearer ${String(config.token ?? "").trim()}`, Accept: "application/json" };
async function get(path: string): Promise<{ status: number; json: unknown }> {
const res = await requestJson(`${base}${path}`, { headers, insecure });
if (res.status === 401) throw new Error("Grafana did not accept the token (HTTP 401) — make a new service account token, and check it has not expired or been deleted");
return { status: res.status, json: res.json };
}
async function must(path: string): Promise<unknown> {
const res = await get(path);
if (res.status === 403) throw new Error(`Grafana says this token may not read ${path.split("?")[0]} (HTTP 403) — give its service account the Viewer role`);
if (res.status === 404) throw new Error("Grafana has no API at this address (HTTP 404) — check the address (its own, not a proxy's)");
if (res.status < 200 || res.status >= 300) throw new Error(`Grafana error: HTTP ${res.status}`);
if (res.json === null) throw new Error("Grafana answered with something that isn't JSON — is the address its own?");
return res.json;
}
const soft = async <T>(run: () => Promise<T>): Promise<T | null> => {
try {
return await run();
} catch {
return null;
}
};
async function read(): Promise<GrafanaRead> {
// The health must work: it says it is there (and the token is tried on what needs it, below).
const health = obj(await must("/api/health"));
const [rules, datasources] = await Promise.all([
soft(async () => parseRules(await must("/api/prometheus/grafana/api/v1/rules"))),
soft(async () => {
const list = parseDatasources(await must("/api/datasources")).slice(0, MAX_DATASOURCES);
return Promise.all(
list.map(async (d): Promise<GrafanaDatasource> => {
const h = await soft(async () => {
const res = await get(`/api/datasources/uid/${encodeURIComponent(d.uid)}/health`);
return res.status >= 200 && res.status < 500 ? parseDatasourceHealth(res.json) : { ok: null, message: null };
});
return { ...d, ...(h ?? { ok: null, message: null }) };
}),
);
}),
]);
if (rules === null && datasources === null) throw new Error("Grafana's health answered, but neither the rules nor the data sources could be read — check the token's permissions");
return { version: str(health.version), rules, datasources };
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
const start = Date.now();
try {
await must("/api/health");
await must("/api/org");
return { ok: true, latencyMs: Date.now() - start };
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) };
}
}
return withDiagLogging("grafana", { ping, read });
}
+6
View File
@@ -25,6 +25,8 @@ import { createHetznerAdapter } from "./hetzner/adapter.js";
import { createSpeedtestAdapter } from "./speedtest/adapter.js";
import { createFragnetAdapter } from "./fragnet/adapter.js";
import { createArrAdapter } from "./arr/adapter.js";
import { createAdguardAdapter } from "./adguard/adapter.js";
import { createGrafanaAdapter } from "./grafana/adapter.js";
import { createNextcloudAdapter } from "./nextcloud/adapter.js";
export interface PingableAdapter {
@@ -90,6 +92,10 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
case "radarr":
case "prowlarr":
return createArrAdapter(type, config as any);
case "adguard":
return createAdguardAdapter(config as any);
case "grafana":
return createGrafanaAdapter(config as any);
case "netbox":
return createNetBoxAdapter(config as any);
case "nextcloud":
+39
View File
@@ -0,0 +1,39 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { adguardState, integrations } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { adguardProblems, adguardSyncStatuses, isAdguardFresh, staleLists, storedOf } from "../services/adguardSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const adguardRouter = Router();
adguardRouter.use(requireAuth);
/** Every AdGuard Home, how it is, from what the last reads stored: whether it runs and protects, whether DNS answers, the statistics, its lists, and what is wrong. Everyone signed in can see it. */
adguardRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, states, statuses] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled, baseUrl: integrations.baseUrl }).from(integrations).where(eq(integrations.type, "adguard")).orderBy(integrations.name),
db.select().from(adguardState),
adguardSyncStatuses(),
]);
const byId = new Map(states.map((s) => [s.integrationId, s]));
const now = Date.now();
res.json({
instances: rows.map((r) => {
const row = byId.get(r.id);
const state = row ? storedOf(row) : null;
const status = statuses[r.id] ?? null;
const fresh = isAdguardFresh(status ?? undefined, now);
return {
id: r.id,
name: r.name,
enabled: r.enabled,
url: r.baseUrl,
status,
fresh,
state: state && { ...state, staleLists: staleLists(state.filters, now).map((l) => l.name) },
problems: state && fresh ? adguardProblems(r.id, r.name, state, now) : [],
};
}),
});
}));
+30
View File
@@ -0,0 +1,30 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { grafanaState, integrations } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { grafanaProblems, grafanaSyncStatuses, isGrafanaFresh, storedOf } from "../services/grafanaSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const grafanaRouter = Router();
grafanaRouter.use(requireAuth);
/** Every Grafana, how it is, from what the last reads stored: which rules are firing or cannot be evaluated, how its data sources test, and what is wrong. Everyone signed in can see it. */
grafanaRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, states, statuses] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled, baseUrl: integrations.baseUrl }).from(integrations).where(eq(integrations.type, "grafana")).orderBy(integrations.name),
db.select().from(grafanaState),
grafanaSyncStatuses(),
]);
const byId = new Map(states.map((s) => [s.integrationId, s]));
const now = Date.now();
res.json({
instances: rows.map((r) => {
const row = byId.get(r.id);
const state = row ? storedOf(row) : null;
const status = statuses[r.id] ?? null;
const fresh = isGrafanaFresh(status ?? undefined, now);
return { id: r.id, name: r.name, enabled: r.enabled, url: r.baseUrl, status, fresh, state, problems: state && fresh ? grafanaProblems(r.id, r.name, state, now) : [] };
}),
});
}));
+20 -2
View File
@@ -40,8 +40,10 @@ import { forgetHetzner, syncHetzner } from "../services/hetznerSync.js";
import { forgetSpeedtest, syncSpeedtest } from "../services/speedtestSync.js";
import { forgetFragnet, syncFragnet } from "../services/fragnetSync.js";
import { forgetArr, syncArr } from "../services/arrSync.js";
import { forgetAdguard, syncAdguard } from "../services/adguardSync.js";
import { forgetGrafana, syncGrafana } from "../services/grafanaSync.js";
import { forgetNextcloud, syncNextcloud } from "../services/nextcloudSync.js";
import { dockerVulnSyncStatuses, findingsOf, forgetDockerVulns, isDockerVulnFresh, syncDockerVulns } from "../services/dockerVulnSync.js";
import { dockerVulnSyncStatuses, findingsOf, forgetDockerVulns, imageAgeDays, imagesOf, isDockerVulnFresh, syncDockerVulns } from "../services/dockerVulnSync.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -155,6 +157,8 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
if (type === "speedtest") void syncSpeedtest(created.id).then(() => invalidateAlertsCache());
if (type === "fragnet") void syncFragnet(created.id).then(() => invalidateAlertsCache());
if (type === "sonarr" || type === "radarr" || type === "prowlarr") void syncArr(created.id).then(() => invalidateAlertsCache());
if (type === "adguard") void syncAdguard(created.id).then(() => invalidateAlertsCache());
if (type === "grafana") void syncGrafana(created.id).then(() => invalidateAlertsCache());
if (type === "nextcloud") void syncNextcloud(created.id).then(() => invalidateAlertsCache());
if (type === "dockhand") void syncDockerVulns(created.id).then(() => invalidateAlertsCache());
@@ -271,6 +275,8 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
if (updated.type === "speedtest" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncSpeedtest(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "fragnet" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncFragnet(updated.id).then(() => invalidateAlertsCache());
if ((updated.type === "sonarr" || updated.type === "radarr" || updated.type === "prowlarr") && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncArr(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "adguard" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncAdguard(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "grafana" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncGrafana(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "nextcloud" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncNextcloud(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "dockhand" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncDockerVulns(updated.id).then(() => invalidateAlertsCache());
@@ -363,6 +369,8 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
if (existing.type === "speedtest") await forgetSpeedtest(id);
if (existing.type === "fragnet") await forgetFragnet(id);
if (existing.type === "sonarr" || existing.type === "radarr" || existing.type === "prowlarr") await forgetArr(id);
if (existing.type === "adguard") await forgetAdguard(id);
if (existing.type === "grafana") await forgetGrafana(id);
if (existing.type === "nextcloud") await forgetNextcloud(id);
if (existing.type === "dockhand") await forgetDockerVulns(id);
await db.delete(integrations).where(eq(integrations.id, id));
@@ -595,6 +603,15 @@ async function requireDockhandAdapter(req: Request, res: Response) {
return { integration: loaded.integration, adapter: createDockhandAdapter(loaded.config as any) };
}
/** The running images of a Docker host by how old they are (the oldest five), and how many are older than the threshold; null when the images were never read. */
function imageAges(row: Parameters<typeof imagesOf>[0] & { imagesAt: string | null }, thresholdDays: number) {
const images = imagesOf(row);
if (images === null) return null;
const now = Date.now();
const dated = images.filter((i): i is typeof i & { created: string } => i.created !== null).map((i) => ({ image: i.image, ageDays: imageAgeDays(i.created, now), containers: i.containers })).sort((a, b) => b.ageDays - a.ageDays || a.image.localeCompare(b.image));
return { at: row.imagesAt, running: images.length, undated: images.length - dated.length, over: thresholdDays > 0 ? dated.filter((i) => i.ageDays > thresholdDays).length : null, oldest: dated.slice(0, 5) };
}
// What Dockhand's own vulnerability scans add up to for each of its Docker hosts, as last read (nothing is asked of Dockhand here, and nothing is scanned).
integrationsRouter.get("/:id/dockhand/vulnerabilities", asyncHandler(async (req, res) => {
const id = Number(req.params.id);
@@ -606,11 +623,12 @@ integrationsRouter.get("/:id/dockhand/vulnerabilities", asyncHandler(async (req,
res.json({
alerts: settings.healthChecks.vulnerabilityAlerts,
highThreshold: settings.healthChecks.vulnerabilityHighThreshold ?? 0,
imageAgeDays: settings.healthChecks.imageAgeDays ?? 0,
status,
fresh: isDockerVulnFresh(status ?? undefined, Date.now()),
environments: rows
.sort((a, b) => a.environmentName.localeCompare(b.environmentName))
.map((r) => ({ environmentId: r.environmentId, environmentName: r.environmentName, critical: r.critical, high: r.high, medium: r.medium, low: r.low, imagesScanned: r.imagesScanned, totalImages: r.totalImages, findings: findingsOf(r), updatedAt: r.updatedAt })),
.map((r) => ({ environmentId: r.environmentId, environmentName: r.environmentName, critical: r.critical, high: r.high, medium: r.medium, low: r.low, imagesScanned: r.imagesScanned, totalImages: r.totalImages, findings: findingsOf(r), images: imageAges(r, settings.healthChecks.imageAgeDays ?? 0), updatedAt: r.updatedAt })),
});
}));
+39 -2
View File
@@ -40,9 +40,9 @@ export function parseDashboardLayout(stored: string | null): DashboardLayout {
meRouter.get("/", requireAuth, asyncHandler(async (req, res) => {
const me = req.currentUser!;
const { id, email, name, role, oidcSub, favorites, dashboardLayout } = me;
const { id, email, name, role, oidcSub, favorites, dashboardLayout, serverPageLayout } = me;
const avatarUrl = avatarUrlFor(me, (await getSettings()).profilePictures);
res.json({ user: { id, sub: oidcSub, email, name, role, avatarUrl, favorites: parseFavorites(favorites), dashboard: parseDashboardLayout(dashboardLayout) } });
res.json({ user: { id, sub: oidcSub, email, name, role, avatarUrl, favorites: parseFavorites(favorites), dashboard: parseDashboardLayout(dashboardLayout), serverPage: parsePageLayout(serverPageLayout) } });
}));
// Paths inside the app only: a slash, then letters, digits, dashes, underscores and further slashes — and, for a saved view of a page,
@@ -68,6 +68,31 @@ const dashboardSchema = z.object({
order: z.record(cardName, z.number().int().min(0).max(1000)).refine((o) => Object.keys(o).length <= MAX_LAYOUT_CARDS, "Too many cards"),
});
export interface PageLayout {
/** Blocks (cards) the person folded: only their header shows. */
folded: string[];
/** Position of a block among the others on the page; lower comes first. */
order: Record<string, number>;
}
const blockName = z.string().regex(/^[a-z0-9-]{1,40}$/);
const pageLayoutSchema = z.object({
folded: z.array(blockName).max(MAX_LAYOUT_CARDS),
order: z.record(blockName, z.number().int().min(0).max(1000)).refine((o) => Object.keys(o).length <= MAX_LAYOUT_CARDS, "Too many blocks"),
});
/** What is stored for a page, read back as a layout. Anything that isn't one is the page as drawn rather than an error. */
export function parsePageLayout(stored: string | null): PageLayout {
const empty: PageLayout = { folded: [], order: {} };
if (!stored) return empty;
try {
const parsed = pageLayoutSchema.safeParse(JSON.parse(stored));
return parsed.success ? parsed.data : empty;
} catch {
return empty;
}
}
// Replaces the signed-in person's pinned pages. Only ever their own.
meRouter.put("/favorites", requireAuth, asyncHandler(async (req, res) => {
const parsed = favoritesSchema.safeParse(req.body);
@@ -104,6 +129,18 @@ meRouter.post("/changes/seen", requireAuth, asyncHandler(async (req, res) => {
res.json({ lastLookedAt: at });
}));
// Replaces how the signed-in person arranged a server's overview page. Only ever their own.
meRouter.put("/server-page", requireAuth, asyncHandler(async (req, res) => {
const parsed = pageLayoutSchema.safeParse(req.body);
if (!parsed.success) {
return res.status(400).json({ error: "invalid_body", message: "Name the blocks to fold and the order of the rest.", details: parsed.error.flatten() });
}
const layout: PageLayout = { folded: [...new Set(parsed.data.folded)], order: parsed.data.order };
const empty = layout.folded.length === 0 && Object.keys(layout.order).length === 0;
await db.update(users).set({ serverPageLayout: empty ? null : JSON.stringify(layout) }).where(eq(users.id, req.currentUser!.id));
res.json({ serverPage: layout });
}));
// Replaces how the signed-in person arranged the dashboard. Only ever their own.
meRouter.put("/dashboard", requireAuth, asyncHandler(async (req, res) => {
const parsed = dashboardSchema.safeParse(req.body);
+1 -1
View File
@@ -144,7 +144,7 @@ const updateSchema = z.object({
.partial()
.optional(),
healthChecks: z
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365), baselineAlerts: z.boolean(), containerAlerts: z.boolean(), vulnerabilityAlerts: z.boolean().default(true), vulnerabilityHighThreshold: z.number().int().min(0).max(10000).default(0), containerIgnore: z.array(z.string().trim().min(1).max(100)).max(200) })
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365), baselineAlerts: z.boolean(), containerAlerts: z.boolean(), vulnerabilityAlerts: z.boolean().default(true), vulnerabilityHighThreshold: z.number().int().min(0).max(10000).default(0), imageAgeDays: z.number().int().min(0).max(3650).default(0), containerIgnore: z.array(z.string().trim().min(1).max(100)).max(200) })
.partial()
.optional(),
quietHours: z
+205
View File
@@ -0,0 +1,205 @@
/**
* Keeps what AdGuard Home says up to date: every two minutes each AdGuard Home integration is read — whether its DNS server runs and protects, the statistics, its filter lists — and one real
* DNS question is put to its DNS server; the state is stored. Pages and alerts are built from what is stored. (Two minutes, not five: DNS is what everything else leans on.)
*
* What is raised: **the DNS server is not running** (AdGuard Home says so: critical); **DNS is not answering** — the test question went unanswered, or was answered with an error, in every read for
* ten minutes (critical; one lost packet is not an outage); **protection is switched off** for an hour or more (someone paused it and forgot: warning); **a filter list has not been updated for a
* week** (warning). "Since when" is what this app saw: the time of the first read that showed it.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { adguardState, integrations } from "../db/schema.js";
import { createAdguardAdapter, type AdguardFilterList, type AdguardRead, type AdguardStats } from "../integrations/adguard/adapter.js";
import type { ProbeResult } from "../integrations/adguard/dnsProbe.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 2 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 10 * 60_000;
const STATUS_FLAG = "adguardSync";
const MIN = 60_000;
const HOUR = 60 * MIN;
const DAY = 24 * HOUR;
/** DNS that has not answered in any read for this long is down. */
export const DNS_FAIL_AFTER_MS = 10 * MIN;
/** Protection off for this long is forgotten, not paused. */
export const PROTECTION_OFF_AFTER_MS = HOUR;
/** A filter list not updated for this long has stopped updating (AdGuard Home updates them daily unless told otherwise). */
export const FILTER_STALE_MS = 7 * DAY;
export interface AdguardSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, AdguardSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, AdguardSyncStatus>) : {};
} catch {
return {};
}
}
export async function adguardSyncStatuses(): Promise<Record<number, AdguardSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isAdguardFresh = (status: AdguardSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof adguardState.$inferSelect;
/** What is stored of one AdGuard Home, as the read gave it, plus what the app has seen between reads. */
export interface StoredAdguard {
version: string | null;
running: boolean;
protectionEnabled: boolean;
/** When the first read that showed protection off was made; null while it is on. */
protectionOffSince: string | null;
dnsPort: number;
stats: AdguardStats | null;
filters: { enabled: boolean; lists: AdguardFilterList[] } | null;
probe: (ProbeResult & { name: string; host: string }) | null;
/** When the first read that showed the DNS test failing was made; null while it works. */
probeFailSince: string | null;
updatedAt: string;
}
function parse<T>(stored: string | null): T | null {
if (stored === null) return null;
try {
return JSON.parse(stored) as T;
} catch {
return null;
}
}
export const storedOf = (row: StateRow): StoredAdguard => ({
version: row.version,
running: row.running,
protectionEnabled: row.protectionEnabled,
protectionOffSince: row.protectionOffSince,
dnsPort: row.dnsPort,
stats: parse<AdguardStats>(row.stats),
filters: parse<StoredAdguard["filters"]>(row.filters),
probe: parse<StoredAdguard["probe"]>(row.probe),
probeFailSince: row.probeFailSince,
updatedAt: row.updatedAt,
});
/** Pure: what the app has seen between reads, carried from the last read. A thing first seen wrong starts its clock now. */
export function carrySince(read: Pick<AdguardRead, "protectionEnabled" | "probe">, previous: Pick<StoredAdguard, "protectionOffSince" | "probeFailSince"> | null, now: Date): { protectionOffSince: string | null; probeFailSince: string | null } {
const at = now.toISOString();
return {
protectionOffSince: read.protectionEnabled ? null : (previous?.protectionOffSince ?? at),
probeFailSince: read.probe.ok ? null : (previous?.probeFailSince ?? at),
};
}
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
export async function storeRead(integrationId: number, r: AdguardRead, now: Date = new Date()): Promise<void> {
const [previous] = await db.select().from(adguardState).where(eq(adguardState.integrationId, integrationId)).limit(1);
const since = carrySince(r, previous ? storedOf(previous) : null, now);
const values = {
version: r.version,
running: r.running,
protectionEnabled: r.protectionEnabled,
protectionOffSince: since.protectionOffSince,
dnsPort: r.dnsPort,
stats: r.stats && JSON.stringify(r.stats),
filters: r.filters && JSON.stringify(r.filters),
probe: JSON.stringify(r.probe),
probeFailSince: since.probeFailSince,
updatedAt: now.toISOString(),
};
await db.insert(adguardState).values({ integrationId, ...values }).onConflictDoUpdate({ target: adguardState.integrationId, set: values });
}
/** Reads one AdGuard Home and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncAdguard(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<AdguardRead>): Promise<AdguardSyncStatus> {
let status: AdguardSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "adguard") throw new Error("not an AdGuard Home integration");
const result = await (read ? read(loaded.config) : createAdguardAdapter(loaded.config as never).read());
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[adguard] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[adguard] couldn't record the result:", err);
}
return status;
}
export async function syncAllAdguard(now: Date = new Date()): Promise<Record<number, AdguardSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "adguard"), eq(integrations.enabled, true)));
const out: Record<number, AdguardSyncStatus> = {};
for (const r of rows) out[r.id] = await syncAdguard(r.id, now);
return out;
}
export async function forgetAdguard(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface AdguardProblem {
key: string;
message: string;
severity: "warning" | "critical";
}
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / DAY), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
/** Pure: the filter lists that are switched on and have not been updated for a week (or never), while filtering is on. */
export function staleLists(filters: StoredAdguard["filters"], nowMs: number): AdguardFilterList[] {
if (filters === null || !filters.enabled) return [];
return filters.lists.filter((l) => l.enabled && (l.updated === null || nowMs - Date.parse(l.updated) > FILTER_STALE_MS));
}
/** Pure: what is wrong with one AdGuard Home, from what is stored for it. */
export function adguardProblems(id: number, name: string, s: StoredAdguard, nowMs: number): AdguardProblem[] {
const out: AdguardProblem[] = [];
if (!s.running) {
out.push({ key: `adguard-stopped:${id}`, severity: "critical", message: `${name}: AdGuard Home says its DNS server is not running — nothing that uses it can look up a name` });
} else if (s.probe && !s.probe.ok && s.probeFailSince !== null && nowMs - Date.parse(s.probeFailSince) >= DNS_FAIL_AFTER_MS) {
out.push({ key: `adguard-dns:${id}`, severity: "critical", message: `${name}: DNS has not answered a question about ${s.probe.name} for ${ago(nowMs - Date.parse(s.probeFailSince))} (asked ${s.probe.host}:${s.dnsPort}) — ${s.probe.error ?? "no answer"}` });
}
if (s.running && !s.protectionEnabled && s.protectionOffSince !== null && nowMs - Date.parse(s.protectionOffSince) >= PROTECTION_OFF_AFTER_MS) {
out.push({ key: `adguard-protection:${id}`, severity: "warning", message: `${name}: AdGuard Home's protection has been switched off for ${ago(nowMs - Date.parse(s.protectionOffSince))} — nothing is being blocked` });
}
const stale = staleLists(s.filters, nowMs);
if (stale.length > 0) {
const names = stale.slice(0, 3).map((l) => l.name);
out.push({ key: `adguard-filters:${id}`, severity: "warning", message: `${name}: ${plural(stale.length, "filter list")} ${stale.length === 1 ? "has" : "have"} not been updated for over a week — ${names.join(", ")}${stale.length > 3 ? ` and ${stale.length - 3} more` : ""}` });
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initAdguardSync(): void {
if (timer) return;
const pass = () => void syncAllAdguard().catch((err) => console.error("[adguard] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 35_000);
console.log(`[adguard] reading AdGuard Home every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+10 -4
View File
@@ -89,6 +89,8 @@ const SOURCES: Record<string, { label: string; link: string }> = {
sonarr: { label: "Sonarr", link: "/media" },
radarr: { label: "Radarr", link: "/media" },
prowlarr: { label: "Prowlarr", link: "/media" },
adguard: { label: "AdGuard Home", link: "/adguard" },
grafana: { label: "Grafana", link: "/grafana" },
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/dns?view=domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
@@ -125,12 +127,16 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "slo", id: Number(m[1]), category: "monitoring" };
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
m = /^docker-vuln(?:-high)?:(\d+):/.exec(key);
m = /^docker-(?:vuln(?:-high)?|image-age):(\d+):/.exec(key);
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
if (m) return { type: "speedtest", id: Number(m[1]), category: "monitoring" };
m = /^hetzner-(off|stuck|rescue|traffic|forecast|backup|lb|cert|fip):(\d+):/.exec(key);
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" || m[1] === "forecast" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
m = /^adguard-(stopped|dns|protection|filters):(\d+)/.exec(key);
if (m) return { type: "adguard", id: Number(m[2]), category: m[1] === "stopped" || m[1] === "dns" ? "services" : "monitoring" };
m = /^grafana-(firing|rule-error|ds):(\d+):/.exec(key);
if (m) return { type: "grafana", id: Number(m[2]), category: m[1] === "ds" ? "integration" : "monitoring" };
m = /^(sonarr|radarr|prowlarr)-(health|queue|disk):(\d+)/.exec(key);
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
@@ -277,7 +283,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : origin.type === "sonarr" || origin.type === "radarr" || origin.type === "prowlarr" ? "/media" : undefined,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : origin.type === "sonarr" || origin.type === "radarr" || origin.type === "prowlarr" ? "/media" : origin.type === "adguard" ? "/adguard" : origin.type === "grafana" ? "/grafana" : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
@@ -287,10 +293,10 @@ export async function collectAlerts(): Promise<AlertsReport> {
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr):(\d+)(?::(.+))?$/.exec(h);
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr|adguard|grafana):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] === "images" ? " (age of the images)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
}
}).finally(() => {
offlineKnown(new Set());
+73 -2
View File
@@ -11,7 +11,7 @@
import { and, eq, notInArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { dockhandVulnerabilities, integrations } from "../db/schema.js";
import { createDockhandAdapter, type DockhandEnvironmentVulns, type DockhandVulnFinding } from "../integrations/dockhand/adapter.js";
import { createDockhandAdapter, type DockhandEnvironmentImages, type DockhandEnvironmentVulns, type DockhandRunningImage, type DockhandVulnFinding } from "../integrations/dockhand/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
@@ -53,6 +53,27 @@ export const findingsOf = (row: Pick<StateRow, "findings">): DockhandVulnFinding
}
};
export const imagesOf = (row: Pick<StateRow, "images">): DockhandRunningImage[] | null => {
if (row.images === null) return null;
try {
const v = JSON.parse(row.images);
return Array.isArray(v) ? (v as DockhandRunningImage[]) : null;
} catch {
return null;
}
};
/** Writes down which images each host's running containers use and when they were built — onto the host's row (a host whose images could not be read keeps what it had). */
export async function storeImages(integrationId: number, environments: DockhandEnvironmentImages[], now: Date = new Date()): Promise<void> {
for (const e of environments) {
if (e.images === null) continue;
await db
.update(dockhandVulnerabilities)
.set({ images: JSON.stringify(e.images), imagesAt: now.toISOString() })
.where(and(eq(dockhandVulnerabilities.integrationId, integrationId), eq(dockhandVulnerabilities.environmentId, e.environmentId)));
}
}
/** Writes down one read, replacing the last one for this integration (a host Dockhand no longer lists is dropped). */
export async function storeRead(integrationId: number, environments: DockhandEnvironmentVulns[], now: Date = new Date()): Promise<void> {
for (const e of environments) {
@@ -74,13 +95,24 @@ export async function storeRead(integrationId: number, environments: DockhandEnv
}
/** Reads one Dockhand's scans and stores them. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncDockerVulns(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<{ environments: DockhandEnvironmentVulns[] }>): Promise<DockerVulnSyncStatus> {
export async function syncDockerVulns(
integrationId: number,
now: Date = new Date(),
read?: (config: Record<string, unknown>) => Promise<{ environments: DockhandEnvironmentVulns[] }>,
readImages?: (config: Record<string, unknown>) => Promise<{ environments: DockhandEnvironmentImages[] }>,
): Promise<DockerVulnSyncStatus> {
let status: DockerVulnSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "dockhand") throw new Error("not a Dockhand integration");
const result = await (read ? read(loaded.config) : createDockhandAdapter(loaded.config as never).readVulnerabilities());
await storeRead(integrationId, result.environments, now);
// The images the containers use (for how old they are) are read beside the scans; a failure here costs only that, not the scans.
try {
await storeImages(integrationId, (await (readImages ? readImages(loaded.config) : createDockhandAdapter(loaded.config as never).readImages())).environments, now);
} catch (err) {
console.error(`[docker-vulns] couldn't read the images of integration ${integrationId}:`, err instanceof Error ? err.message : err);
}
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
@@ -148,6 +180,45 @@ export function dockerVulnProblems(integrationId: number, rows: Pick<StateRow, "
return out;
}
export interface DockerImageProblem {
key: string;
message: string;
severity: "warning";
}
const DAY = 86_400_000;
export const imageAgeDays = (created: string, nowMs: number): number => Math.floor((nowMs - Date.parse(created)) / DAY);
/** Pure: the running images built more than `thresholdDays` ago, oldest first (an image whose build date is not known is not judged). */
export function oldImages(images: DockhandRunningImage[] | null, thresholdDays: number, nowMs: number): { image: string; ageDays: number; containers: string[] }[] {
if (images === null || thresholdDays <= 0) return [];
return images
.filter((i): i is DockhandRunningImage & { created: string } => i.created !== null)
.map((i) => ({ image: i.image, ageDays: imageAgeDays(i.created, nowMs), containers: i.containers }))
.filter((i) => i.ageDays > thresholdDays)
.sort((a, b) => b.ageDays - a.ageDays || a.image.localeCompare(b.image));
}
/**
* Pure: a warning for each Docker host that has running images built more than `thresholdDays` ago — however the scans of them came out (an image that was never scanned, or scanned clean, is just as
* stale). 0 turns it off. Judged from the stored list; a list older than STALE_AFTER_MS or never read says nothing (the caller holds it).
*/
export function dockerImageAgeProblems(integrationId: number, rows: Pick<StateRow, "environmentId" | "environmentName" | "images">[], thresholdDays: number, nowMs: number): DockerImageProblem[] {
const out: DockerImageProblem[] = [];
if (thresholdDays <= 0) return out;
for (const r of [...rows].sort((a, b) => a.environmentName.localeCompare(b.environmentName))) {
const old = oldImages(imagesOf(r), thresholdDays, nowMs);
if (old.length === 0) continue;
const named = old.slice(0, 3).map((i) => `${i.image} (${i.ageDays} days)`);
out.push({
key: `docker-image-age:${integrationId}:${r.environmentId}`,
severity: "warning",
message: `${r.environmentName}: ${old.length} running ${old.length === 1 ? "image was" : "images were"} built more than ${thresholdDays} days ago — ${named.join(", ")}${old.length > 3 ? ` and ${old.length - 3} more` : ""}`,
});
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initDockerVulnSync(): void {
+158
View File
@@ -0,0 +1,158 @@
/**
* Keeps what Grafana says up to date: every five minutes each Grafana integration is read — its alert rules (which are firing, which cannot be evaluated) and its data sources (with Grafana's own
* health check of each) — and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Grafana.
*
* What is raised: **a rule that is firing** (critical when the rule is labelled severity critical or page, a warning otherwise — a pending one is shown, not raised), **a rule Grafana cannot
* evaluate** (an alert that cannot alert is worse than a quiet one), and **a data source that fails its health check**. Grafana can notify about its own alerts too; with both you hear twice, so
* this is for the ones that do not reach you any other way, or for having them in one list.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { grafanaState, integrations } from "../db/schema.js";
import { createGrafanaAdapter, type GrafanaDatasource, type GrafanaRead, type GrafanaRule } from "../integrations/grafana/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 5 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 20 * 60_000;
const STATUS_FLAG = "grafanaSync";
const MIN = 60_000;
const HOUR = 60 * MIN;
const DAY = 24 * HOUR;
export interface GrafanaSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, GrafanaSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, GrafanaSyncStatus>) : {};
} catch {
return {};
}
}
export async function grafanaSyncStatuses(): Promise<Record<number, GrafanaSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isGrafanaFresh = (status: GrafanaSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof grafanaState.$inferSelect;
export interface StoredGrafana {
version: string | null;
/** Null when the rules could not be read. */
rules: { total: number; items: GrafanaRule[] } | null;
datasources: GrafanaDatasource[] | null;
updatedAt: string;
}
function parse<T>(stored: string | null): T | null {
if (stored === null) return null;
try {
return JSON.parse(stored) as T;
} catch {
return null;
}
}
export const storedOf = (row: StateRow): StoredGrafana => ({ version: row.version, rules: parse<StoredGrafana["rules"]>(row.rules), datasources: parse<GrafanaDatasource[]>(row.datasources), updatedAt: row.updatedAt });
/** Writes down one read, replacing the last. */
export async function storeRead(integrationId: number, r: GrafanaRead, now: Date = new Date()): Promise<void> {
const values = { version: r.version, rules: r.rules && JSON.stringify(r.rules), datasources: r.datasources && JSON.stringify(r.datasources), updatedAt: now.toISOString() };
await db.insert(grafanaState).values({ integrationId, ...values }).onConflictDoUpdate({ target: grafanaState.integrationId, set: values });
}
/** Reads one Grafana and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncGrafana(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<GrafanaRead>): Promise<GrafanaSyncStatus> {
let status: GrafanaSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "grafana") throw new Error("not a Grafana integration");
const result = await (read ? read(loaded.config) : createGrafanaAdapter(loaded.config as never).read());
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[grafana] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[grafana] couldn't record the result:", err);
}
return status;
}
export async function syncAllGrafana(now: Date = new Date()): Promise<Record<number, GrafanaSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "grafana"), eq(integrations.enabled, true)));
const out: Record<number, GrafanaSyncStatus> = {};
for (const r of rows) out[r.id] = await syncGrafana(r.id, now);
return out;
}
export async function forgetGrafana(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface GrafanaProblem {
key: string;
message: string;
severity: "warning" | "critical";
}
const slug = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48);
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / DAY), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
export const isCriticalRule = (r: Pick<GrafanaRule, "severity">) => r.severity === "critical" || r.severity === "page";
/** Pure: what is wrong with one Grafana, from what is stored for it. */
export function grafanaProblems(id: number, name: string, s: StoredGrafana, nowMs: number): GrafanaProblem[] {
const out: GrafanaProblem[] = [];
for (const r of [...(s.rules?.items ?? [])].sort((a, b) => a.name.localeCompare(b.name))) {
const ident = slug(r.uid ?? `${r.group}-${r.name}`);
const where = [r.folder, r.group].filter(Boolean).join(" / ");
if (r.state === "firing") {
out.push({
key: `grafana-firing:${id}:${ident}`,
severity: isCriticalRule(r) ? "critical" : "warning",
message: `${name}: the rule "${r.name}" is firing${where ? ` (${where})` : ""}${r.activeAt ? ` — for ${ago(nowMs - Date.parse(r.activeAt))}` : ""}${r.summary ? ` — ${r.summary}` : ""}`,
});
}
if (r.health === "error") {
out.push({ key: `grafana-rule-error:${id}:${ident}`, severity: "warning", message: `${name}: Grafana cannot evaluate the rule "${r.name}"${where ? ` (${where})` : ""} — ${r.lastError ?? "it reports an error"}` });
}
}
for (const d of [...(s.datasources ?? [])].sort((a, b) => a.name.localeCompare(b.name))) {
if (d.ok === false) out.push({ key: `grafana-ds:${id}:${slug(d.uid)}`, severity: "warning", message: `${name}: the data source ${d.name} (${d.type}) fails Grafana's health check${d.message ? ` — ${d.message}` : ""}` });
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initGrafanaSync(): void {
if (timer) return;
const pass = () => void syncAllGrafana().catch((err) => console.error("[grafana] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 39_000);
console.log(`[grafana] reading Grafana every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+68 -2
View File
@@ -1,6 +1,6 @@
import { and, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState } from "../db/schema.js";
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState , adguardState , grafanaState } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
@@ -28,10 +28,12 @@ import { isPaperlessFresh, paperlessProblems, paperlessSyncStatuses, type Paperl
import { isN8nFresh, n8nProblems, n8nSyncStatuses, type N8nProblem } from "./n8nSync.js";
import { isNetBoxFresh, netboxProblems, netboxSyncStatuses, type NetBoxProblem } from "./netboxSync.js";
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type NextcloudProblem } from "./nextcloudSync.js";
import { dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, type DockerVulnProblem } from "./dockerVulnSync.js";
import { dockerImageAgeProblems, dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, STALE_AFTER_MS as DOCKER_VULN_STALE_MS, type DockerImageProblem, type DockerVulnProblem } from "./dockerVulnSync.js";
import { expectedOf as speedtestExpected, isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestProblems, speedtestSyncStatuses, type SpeedtestProblem } from "./speedtestSync.js";
import { hetznerProblems, hetznerSyncStatuses, imagesOf as hetznerImagesOf, loadBalancersOf as hetznerLoadBalancersOf, floatingIpsOf as hetznerFloatingIpsOf, certificatesOf as hetznerCertificatesOf, isHetznerFresh, serversOf as hetznerServersOf, type HetznerProblem } from "./hetznerSync.js";
import { ARR_KINDS, type ArrKind } from "../integrations/arr/adapter.js";
import { adguardProblems, adguardSyncStatuses, isAdguardFresh, storedOf as adguardStoredOf, type AdguardProblem } from "./adguardSync.js";
import { grafanaProblems, grafanaSyncStatuses, isGrafanaFresh, storedOf as grafanaStoredOf, type GrafanaProblem } from "./grafanaSync.js";
import { arrProblems, arrSyncStatuses, isArrFresh, storedOf as arrStoredOf, type ArrProblem } from "./arrSync.js";
import { fragnetProblems, fragnetSyncStatuses, isFragnetFresh, optionsOf as fragnetOptions, serversOf as fragnetServersOf, type FragnetProblem } from "./fragnetSync.js";
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
@@ -124,6 +126,8 @@ export interface HealthSnapshot {
dockerMemory?: ContainerMemory;
/** Critical vulnerabilities in the images of each Dockhand's Docker hosts, from its last scans, only while that reading is recent (services/dockerVulnSync.ts). */
dockerVulns?: { integrationId: number; problems: DockerVulnProblem[] }[];
/** Docker hosts with running images built long ago, only while the list of images is recent and the setting is on (services/dockerVulnSync.ts). */
dockerImages?: { integrationId: number; problems: DockerImageProblem[] }[];
/** What is wrong with each Hetzner Cloud project (a server that went off, a traffic quota nearly used, rescue mode left on, a missing backup), only while that reading is recent (services/hetznerSync.ts). */
hetzner?: { integrationId: number; problems: HetznerProblem[] }[];
/** What is wrong with each Speedtest Tracker (the internet slower than it is sold at, speed tests that keep failing), only while that reading is recent (services/speedtestSync.ts). */
@@ -132,6 +136,10 @@ export interface HealthSnapshot {
fragnet?: { integrationId: number; problems: FragnetProblem[] }[];
/** What is wrong with each Sonarr, Radarr and Prowlarr (what it says itself, downloads that are stuck, a disk nearly full), only while that reading is recent (services/arrSync.ts). */
arr?: { integrationId: number; kind: ArrKind; problems: ArrProblem[] }[];
/** What is wrong with each AdGuard Home (DNS not running or not answering, protection left off, filter lists not updating), only while that reading is recent (services/adguardSync.ts). */
adguard?: { integrationId: number; problems: AdguardProblem[] }[];
/** What is wrong with each Grafana (rules firing or failing, data sources failing their check), only while that reading is recent (services/grafanaSync.ts). */
grafana?: { integrationId: number; problems: GrafanaProblem[] }[];
/** Containers to leave alone. */
dockerIgnore?: string[];
/** The clock the container judgements are made on (the pass's, so a test can set it). */
@@ -273,6 +281,10 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
// Speedtest Tracker: the internet slower than it is sold at, speed tests that keep failing.
for (const s of snapshot.speedtest ?? []) for (const p of s.problems) out.push({ key: p.key, source: `speedtest:${s.integrationId}`, message: p.message, severity: p.severity });
// AdGuard Home and Grafana.
for (const a of snapshot.adguard ?? []) for (const p of a.problems) out.push({ key: p.key, source: `adguard:${a.integrationId}`, message: p.message, severity: p.severity });
for (const g of snapshot.grafana ?? []) for (const p of g.problems) out.push({ key: p.key, source: `grafana:${g.integrationId}`, message: p.message, severity: p.severity });
// Sonarr, Radarr and Prowlarr: what they say is wrong, downloads that are stuck, disks nearly full.
for (const a of snapshot.arr ?? []) for (const p of a.problems) out.push({ key: p.key, source: `${a.kind}:${a.integrationId}`, message: p.message, severity: p.severity });
@@ -284,6 +296,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
// Critical vulnerabilities in the images Dockhand has scanned.
for (const v of snapshot.dockerVulns ?? []) for (const p of v.problems) out.push({ key: p.key, source: `dockhand:${v.integrationId}:vuln`, message: p.message, severity: p.severity });
for (const v of snapshot.dockerImages ?? []) for (const p of v.problems) out.push({ key: p.key, source: `dockhand:${v.integrationId}:images`, message: p.message, severity: p.severity });
// A service-level target that has used most of its month's error budget, or all of it.
for (const f of snapshot.levels ?? []) out.push({ key: levelKey(f), source: "slo", message: f.message, severity: f.severity });
@@ -544,6 +557,42 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
console.error("[health] couldn't judge Speedtest Tracker:", err instanceof Error ? err.message : err);
}
// AdGuard Home: read by its own pass every two minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsG = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "adguard"), eq(integrations.enabled, true)));
if (rowsG.length > 0) {
const statuses = await adguardSyncStatuses();
for (const r of rowsG) {
const [state] = await db.select().from(adguardState).where(eq(adguardState.integrationId, r.id)).limit(1);
if (!state || !isAdguardFresh(statuses[r.id], Date.now())) {
held.add(`adguard:${r.id}`);
continue;
}
(snapshot.adguard ??= []).push({ integrationId: r.id, problems: adguardProblems(r.id, r.name, adguardStoredOf(state), Date.now()) });
}
}
} catch (err) {
console.error("[health] couldn't judge AdGuard Home:", err instanceof Error ? err.message : err);
}
// Grafana: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsR = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "grafana"), eq(integrations.enabled, true)));
if (rowsR.length > 0) {
const statuses = await grafanaSyncStatuses();
for (const r of rowsR) {
const [state] = await db.select().from(grafanaState).where(eq(grafanaState.integrationId, r.id)).limit(1);
if (!state || !isGrafanaFresh(statuses[r.id], Date.now())) {
held.add(`grafana:${r.id}`);
continue;
}
(snapshot.grafana ??= []).push({ integrationId: r.id, problems: grafanaProblems(r.id, r.name, grafanaStoredOf(state), Date.now()) });
}
}
} catch (err) {
console.error("[health] couldn't judge Grafana:", err instanceof Error ? err.message : err);
}
// Sonarr, Radarr and Prowlarr: read by their own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsA = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(and(inArray(integrations.type, ARR_KINDS), eq(integrations.enabled, true)));
@@ -768,6 +817,23 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
}
}
// How old the images the containers run are, as the same sync pass stored them (only with the setting on; a list gone stale is held, not cleared).
if (checks.imageAgeDays > 0) {
try {
for (const row of await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "dockhand"), eq(integrations.enabled, true)))) {
const stored = await db.select().from(dockhandVulnerabilities).where(eq(dockhandVulnerabilities.integrationId, row.id));
const fresh = stored.filter((r) => r.images !== null && r.imagesAt !== null && Date.now() - Date.parse(r.imagesAt) <= DOCKER_VULN_STALE_MS);
if (stored.length === 0 || fresh.length < stored.filter((r) => r.images !== null).length || fresh.length === 0) {
held.add(`dockhand:${row.id}:images`);
continue;
}
(snapshot.dockerImages ??= []).push({ integrationId: row.id, problems: dockerImageAgeProblems(row.id, fresh, checks.imageAgeDays, Date.now()) });
}
} catch (err) {
console.error("[health] couldn't judge the age of the images:", err instanceof Error ? err.message : err);
}
}
// Service-level targets: how much of the month's error budget is used. A failure here costs only this check.
try {
snapshot.levels = await levelFindings();
+2 -2
View File
@@ -6,7 +6,7 @@ export const INTEGRATION_GROUPS: { id: string; label: string; types: string[] }[
{ id: "infrastructure", label: "Infrastructure", types: ["proxmox", "synology", "dockhand", "hetzner", "fragnet", "tailscale"] },
{ id: "apps", label: "Apps", types: ["homeassistant", "jellyfin", "sonarr", "radarr", "prowlarr", "immich", "paperless", "nextcloud", "authentik"] },
{ id: "backups", label: "Backups", types: ["pbs"] },
{ id: "network", label: "Network", types: ["unifi", "npm", "phpipam", "netbox"] },
{ id: "monitoring", label: "Monitoring", types: ["uptimekuma", "prometheus", "ups", "speedtest"] },
{ id: "network", label: "Network", types: ["unifi", "npm", "phpipam", "netbox", "adguard"] },
{ id: "monitoring", label: "Monitoring", types: ["uptimekuma", "prometheus", "grafana", "ups", "speedtest"] },
{ id: "automation", label: "Automation", types: ["semaphore", "gitea", "osticket", "n8n"] },
];
+5 -1
View File
@@ -31,7 +31,7 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `server:${m[1]}`;
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^docker-vuln(?:-high)?:(\d+):/.exec(key);
m = /^docker-(?:vuln(?:-high)?|image-age):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
@@ -45,6 +45,10 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `integration:${m[1]}`;
m = /^(?:sonarr|radarr|prowlarr)-(?:health|queue|disk):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^adguard-(?:stopped|dns|protection|filters):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^grafana-(?:firing|rule-error|ds):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^netbox-(?:workers):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^n8n-(?:failed):(\d+)/.exec(key);
+46 -1
View File
@@ -5,7 +5,7 @@
*/
import { eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState } from "../db/schema.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState, adguardState, grafanaState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { backupState, listAll as listDuplicati } from "./duplicati.js";
@@ -15,6 +15,8 @@ import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf, loadBalancersOf as hetznerLoadBalancersOf, certificatesOf as hetznerCertificatesOf, trafficForecast as hetznerTrafficForecast } from "./hetznerSync.js";
import { ARR_KINDS } from "../integrations/arr/adapter.js";
import { adguardSyncStatuses, isAdguardFresh, storedOf as adguardStoredOf } from "./adguardSync.js";
import { grafanaSyncStatuses, isGrafanaFresh, storedOf as grafanaStoredOf } from "./grafanaSync.js";
import { arrSyncStatuses, distinctDisks as arrDisks, isArrFresh, stuckItems as arrStuck, storedOf as arrStoredOf } from "./arrSync.js";
import { fragnetSyncStatuses, isFragnetFresh, newestBackup as fragnetNewestBackup, serversOf as fragnetServersOf } from "./fragnetSync.js";
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
@@ -103,6 +105,47 @@ async function addSpeedtestMetrics(out: Exposition, now: Date) {
}
}
/** Each AdGuard Home, for /metrics: whether its DNS server runs and protects, whether the test question was answered and how fast, and what it has done over its statistics period. Only while the reading is recent. */
async function addAdguardMetrics(out: Exposition, now: Date) {
const statuses = await adguardSyncStatuses();
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "adguard"))).map((r) => [r.id, r]));
for (const s of await db.select().from(adguardState)) {
const row = rows.get(s.integrationId);
if (!row || !row.enabled || !isAdguardFresh(statuses[s.integrationId], now.getTime())) continue;
const state = adguardStoredOf(s);
const labels = { name: row.name };
out.add("homelab_adguard_running", "gauge", "1 if AdGuard Home's DNS server is running, else 0.", state.running ? 1 : 0, labels);
out.add("homelab_adguard_protection_enabled", "gauge", "1 if AdGuard Home's protection is switched on, else 0.", state.protectionEnabled ? 1 : 0, labels);
if (state.probe) {
out.add("homelab_adguard_dns_up", "gauge", "1 if AdGuard Home's DNS server answered a test question, else 0.", state.probe.ok ? 1 : 0, labels);
if (state.probe.ok && state.probe.ms !== null) out.add("homelab_adguard_dns_probe_seconds", "gauge", "How long AdGuard Home's DNS server took to answer a test question, in seconds.", state.probe.ms / 1000, labels);
}
if (state.stats) {
out.add("homelab_adguard_queries", "gauge", "DNS queries AdGuard Home answered in its statistics period.", state.stats.queries, labels);
out.add("homelab_adguard_blocked", "gauge", "Of them, the ones it blocked.", state.stats.blocked, labels);
if (state.stats.avgMs !== null) out.add("homelab_adguard_avg_processing_seconds", "gauge", "Average time AdGuard Home takes to process a query, in seconds.", state.stats.avgMs / 1000, labels);
}
}
}
/** Each Grafana, for /metrics: how many rules are firing, pending and cannot be evaluated, and how many data sources fail their health check. Only while the reading is recent. */
async function addGrafanaMetrics(out: Exposition, now: Date) {
const statuses = await grafanaSyncStatuses();
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "grafana"))).map((r) => [r.id, r]));
for (const s of await db.select().from(grafanaState)) {
const row = rows.get(s.integrationId);
if (!row || !row.enabled || !isGrafanaFresh(statuses[s.integrationId], now.getTime())) continue;
const state = grafanaStoredOf(s);
const labels = { name: row.name };
if (state.rules) {
out.add("homelab_grafana_rules", "gauge", "How many alert rules Grafana has.", state.rules.total, labels);
out.add("homelab_grafana_rules_firing", "gauge", "How many of Grafana's alert rules are firing.", state.rules.items.filter((r) => r.state === "firing").length, labels);
out.add("homelab_grafana_rules_error", "gauge", "How many of Grafana's alert rules cannot be evaluated.", state.rules.items.filter((r) => r.health === "error").length, labels);
}
if (state.datasources) out.add("homelab_grafana_datasources_failing", "gauge", "How many of Grafana's data sources fail its health check.", state.datasources.filter((d) => d.ok === false).length, labels);
}
}
/** Each Sonarr, Radarr and Prowlarr, for /metrics: how many health findings it has, its queue and how much of it is stuck, the missing items and the free space of each disk. Only while the reading is recent. */
async function addArrMetrics(out: Exposition, now: Date) {
const statuses = await arrSyncStatuses();
@@ -498,6 +541,8 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addSpeedtestMetrics(out, now);
await addFragnetMetrics(out, now);
await addArrMetrics(out, now);
await addAdguardMetrics(out, now);
await addGrafanaMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}
+3 -1
View File
@@ -146,6 +146,8 @@ export interface HealthCheckSettings {
vulnerabilityAlerts: boolean;
/** A Docker host whose scanned images have at least this many HIGH findings is a warning too (0 = off, the default; critical ones are alerted whatever this is, while vulnerabilityAlerts is on). */
vulnerabilityHighThreshold: number;
/** A Docker host with running images built more than this many days ago is a warning (0 = off, the default) — however their scans came out (services/dockerVulnSync.ts). */
imageAgeDays: number;
/** Containers to leave alone: a name, or "host/name"; * stands for anything. */
containerIgnore: string[];
}
@@ -331,7 +333,7 @@ const DEFAULTS: AppSettings = {
selfBackup: { enabled: true, time: "03:30", keep: 14 },
profilePictures: { gravatar: false, identityProvider: false },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, vulnerabilityAlerts: true, vulnerabilityHighThreshold: 0, containerIgnore: [] },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, vulnerabilityAlerts: true, vulnerabilityHighThreshold: 0, imageAgeDays: 0, containerIgnore: [] },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
consistency: { excludedRanges: ["172.16.0.0/12"] },