AdGuard Home, Grafana, Docker image age, foldable server page cards
New integration kinds adguard (status, statistics, filter lists and a real DNS question over UDP; alerts for DNS down/not running, protection left off, stale lists) and grafana (alert rules firing or failing, data-source health). Optional warning for running Docker images built more than N days ago (off by default). A server's overview cards can be folded and arranged, kept per user. Migrations 0075-0077; docs, metrics and tests included. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3d9a2310bb
commit
551eb1597a
53 files changed
+27700
-33
No files matched your search
@@ -0,0 +1 @@
|
||||
ALTER TABLE `users` ADD `server_page_layout` text;
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE `dockhand_vulnerabilities` ADD `images` text;--> statement-breakpoint
|
||||
ALTER TABLE `dockhand_vulnerabilities` ADD `images_at` text;
|
||||
@@ -0,0 +1,23 @@
|
||||
CREATE TABLE `adguard_state` (
|
||||
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||
`version` text,
|
||||
`running` integer DEFAULT true NOT NULL,
|
||||
`protection_enabled` integer DEFAULT true NOT NULL,
|
||||
`protection_off_since` text,
|
||||
`dns_port` integer DEFAULT 53 NOT NULL,
|
||||
`stats` text,
|
||||
`filters` text,
|
||||
`probe` text,
|
||||
`probe_fail_since` text,
|
||||
`updated_at` text NOT NULL,
|
||||
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE `grafana_state` (
|
||||
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||
`version` text,
|
||||
`rules` text,
|
||||
`datasources` text,
|
||||
`updated_at` text NOT NULL,
|
||||
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -526,6 +526,27 @@
|
||||
"when": 1793128598267,
|
||||
"tag": "0074_open_the_hood",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 75,
|
||||
"version": "6",
|
||||
"when": 1793134285016,
|
||||
"tag": "0075_rare_morlun",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 76,
|
||||
"version": "6",
|
||||
"when": 1793134583724,
|
||||
"tag": "0076_short_mordo",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 77,
|
||||
"version": "6",
|
||||
"when": 1793134873418,
|
||||
"tag": "0077_opposite_catseye",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -71,6 +71,8 @@ import { speedtestRouter } from "./routes/speedtest.js";
|
||||
import { fragnetRouter } from "./routes/fragnet.js";
|
||||
import { arrRouter } from "./routes/arr.js";
|
||||
import { wallRouter } from "./routes/wall.js";
|
||||
import { adguardRouter } from "./routes/adguard.js";
|
||||
import { grafanaRouter } from "./routes/grafana.js";
|
||||
import { costsRouter } from "./routes/costs.js";
|
||||
import { nextcloudRouter } from "./routes/nextcloud.js";
|
||||
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
|
||||
@@ -220,6 +222,8 @@ export function createApp() {
|
||||
app.use("/api/fragnet", fragnetRouter);
|
||||
app.use("/api/arr", arrRouter);
|
||||
app.use("/api/wall", wallRouter);
|
||||
app.use("/api/adguard", adguardRouter);
|
||||
app.use("/api/grafana", grafanaRouter);
|
||||
app.use("/api/costs", costsRouter);
|
||||
app.use("/api/nextcloud", nextcloudRouter);
|
||||
app.use("/api/service-levels", serviceLevelsRouter);
|
||||
|
||||
@@ -26,6 +26,9 @@ export const users = sqliteTable("users", {
|
||||
// How this person arranged the dashboard: a JSON object with the cards they hid ("hidden": ["Gitea"]) and the order of the others
|
||||
// ("order": {"DNS": 0, "Secrets": 1}). Follows them to any browser; null is the default layout.
|
||||
dashboardLayout: text("dashboard_layout"),
|
||||
// How this person arranged a server's overview page: the cards they folded ("folded": ["admin-links"]) and the order of the blocks ("order": {"notes": 10}). The same for every server;
|
||||
// follows them to any browser; null is the page as drawn.
|
||||
serverPageLayout: text("server_page_layout"),
|
||||
// The address of the picture the identity provider gave for this person (the OIDC "picture" claim), as of their last sign-in. Only an address:
|
||||
// nothing is fetched from it unless Settings → Sign-in → Profile pictures allows it, and then only by this server, never by the browser.
|
||||
pictureUrl: text("picture_url"),
|
||||
@@ -567,6 +570,8 @@ export const integrationTypes = [
|
||||
"sonarr",
|
||||
"radarr",
|
||||
"prowlarr",
|
||||
"adguard",
|
||||
"grafana",
|
||||
] as const;
|
||||
export type IntegrationType = (typeof integrationTypes)[number];
|
||||
|
||||
@@ -1206,6 +1211,8 @@ export const dockhandVulnerabilities = sqliteTable(
|
||||
imagesScanned: integer("images_scanned"),
|
||||
totalImages: integer("total_images"),
|
||||
findings: text("findings").notNull().default("[]"), // JSON: the critical findings (cve, package, installedVersion, fixedVersion, imageName, containers), at most 100
|
||||
images: text("images"), // JSON: the images the running containers of this host use, each with when it was built (image, created, containers); null: never read, or could not be
|
||||
imagesAt: text("images_at"), // ISO: when `images` was last read
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.integrationId, t.environmentId] })],
|
||||
@@ -1322,6 +1329,41 @@ export const arrState = sqliteTable("arr_state", {
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── AdGuard Home ───────────────────────────────────────────────────────────
|
||||
|
||||
// How an AdGuard Home (an integration of type "adguard") is right now, one row each, replaced at every read: whether its DNS server runs and protects, the statistics, its filter lists and the result
|
||||
// of a real DNS question put to it — plus since when protection has been off and the question unanswered, which only this app can know. A column that is null could not be read. Read by
|
||||
// services/adguardSync.ts.
|
||||
export const adguardState = sqliteTable("adguard_state", {
|
||||
integrationId: integer("integration_id")
|
||||
.primaryKey()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
version: text("version"),
|
||||
running: integer("running", { mode: "boolean" }).notNull().default(true),
|
||||
protectionEnabled: integer("protection_enabled", { mode: "boolean" }).notNull().default(true),
|
||||
protectionOffSince: text("protection_off_since"), // ISO: the first read that showed protection off; null while it is on
|
||||
dnsPort: integer("dns_port").notNull().default(53),
|
||||
stats: text("stats"), // JSON: queries, blocked, average time, top names and clients; null: could not be read
|
||||
filters: text("filters"), // JSON: { enabled, lists: [{ name, enabled, rules, updated }] }; null: could not be read
|
||||
probe: text("probe"), // JSON: the DNS question — { ok, ms, error, name, host }
|
||||
probeFailSince: text("probe_fail_since"), // ISO: the first read in which the question went unanswered; null while it is answered
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── Grafana ────────────────────────────────────────────────────────────────
|
||||
|
||||
// How a Grafana (an integration of type "grafana") is right now, one row each, replaced at every read: the alert rules that are firing, pending or cannot be evaluated (and how many there are), and
|
||||
// its data sources with the result of Grafana's own health check of each. A column that is null could not be read. Read by services/grafanaSync.ts.
|
||||
export const grafanaState = sqliteTable("grafana_state", {
|
||||
integrationId: integer("integration_id")
|
||||
.primaryKey()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
version: text("version"),
|
||||
rules: text("rules"), // JSON { total, items }; null: could not be read
|
||||
datasources: text("datasources"), // JSON; null: could not be listed
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── NetBox ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// How a NetBox (an integration of type "netbox") is right now, one row each, replaced at every read: its version, how many background workers are running, the counts and the IP
|
||||
|
||||
@@ -42,6 +42,8 @@ import { initHetznerSync } from "./services/hetznerSync.js";
|
||||
import { initSpeedtestSync } from "./services/speedtestSync.js";
|
||||
import { initFragnetSync } from "./services/fragnetSync.js";
|
||||
import { initArrSync } from "./services/arrSync.js";
|
||||
import { initAdguardSync } from "./services/adguardSync.js";
|
||||
import { initGrafanaSync } from "./services/grafanaSync.js";
|
||||
import { backfillMetricRollups } from "./services/serverMetrics.js";
|
||||
import { backfillEndpointHourly } from "./services/endpointChecks.js";
|
||||
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
|
||||
@@ -105,6 +107,8 @@ initHetznerSync();
|
||||
initSpeedtestSync();
|
||||
initFragnetSync();
|
||||
initArrSync();
|
||||
initAdguardSync();
|
||||
initGrafanaSync();
|
||||
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
|
||||
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
|
||||
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
/**
|
||||
* AdGuard Home adapter. Config: url (its web interface), username and password (its own sign-in — HTTP Basic; leave them empty if it has none), optionally dnsHost (where its DNS server is, when
|
||||
* that is not the host of the web address — a reverse proxy in front of the interface, say), probeDomain (the name a test question is asked about; example.com unless told) and insecure.
|
||||
*
|
||||
* Reading only: whether it runs and protects (`GET /control/status`), what it has done in the statistics period (`GET /control/stats`: queries, how many blocked, how long an answer takes, the
|
||||
* names blocked most and the clients asking most), its filter lists and when each was last updated (`GET /control/filtering/status`) — and one real DNS question put to its DNS server over UDP
|
||||
* (see dnsProbe.ts), because the interface can be up while DNS is not. Nothing is changed: protection is never switched, no list refreshed.
|
||||
*
|
||||
* Endpoints and fields are from AdGuard Home's published OpenAPI file (openapi/openapi.yaml: ServerStatus, Stats, FilterStatus) and have not been checked against a live install here; every
|
||||
* field is read defensively.
|
||||
*/
|
||||
import { requestJson } from "../httpJson.js";
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
import { probeDns, type ProbeResult } from "./dnsProbe.js";
|
||||
|
||||
export interface AdguardConfig {
|
||||
url: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
insecure?: boolean;
|
||||
dnsHost?: string;
|
||||
probeDomain?: string;
|
||||
}
|
||||
|
||||
export const DEFAULT_PROBE_DOMAIN = "example.com";
|
||||
|
||||
export interface AdguardCounted {
|
||||
name: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
export interface AdguardStats {
|
||||
/** Over AdGuard Home's statistics period (a day unless it was set otherwise): see `periodHours`. */
|
||||
queries: number;
|
||||
blocked: number;
|
||||
/** Blocked as a share of the queries, in percent; null with no queries. */
|
||||
blockedPercent: number | null;
|
||||
/** Average time to answer, in milliseconds; null when not said. */
|
||||
avgMs: number | null;
|
||||
/** Of the blocked ones, the ones made safe by Safe Browsing, parental control and safe search. */
|
||||
safeBrowsing: number;
|
||||
parental: number;
|
||||
safeSearch: number;
|
||||
periodHours: number | null;
|
||||
topBlocked: AdguardCounted[];
|
||||
topClients: AdguardCounted[];
|
||||
topQueried: AdguardCounted[];
|
||||
}
|
||||
|
||||
export interface AdguardFilterList {
|
||||
name: string;
|
||||
enabled: boolean;
|
||||
rules: number | null;
|
||||
/** ISO; null when it has never been updated. */
|
||||
updated: string | null;
|
||||
}
|
||||
|
||||
export interface AdguardRead {
|
||||
version: string | null;
|
||||
/** Whether its DNS server is running. */
|
||||
running: boolean;
|
||||
protectionEnabled: boolean;
|
||||
dnsPort: number;
|
||||
/** Null when the statistics could not be read. */
|
||||
stats: AdguardStats | null;
|
||||
/** Null when the filter lists could not be read. */
|
||||
filters: { enabled: boolean; lists: AdguardFilterList[] } | null;
|
||||
/** The real DNS question: whether it was answered, how fast, and about what. */
|
||||
probe: ProbeResult & { name: string; host: string };
|
||||
}
|
||||
|
||||
export interface AdguardAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(): Promise<AdguardRead>;
|
||||
}
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
|
||||
const arr = (v: unknown): unknown[] => (Array.isArray(v) ? v : []);
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
|
||||
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||||
|
||||
/** Pure: AdGuard's lists of counts — `[{ "example.com": 12 }, { "ads.example": 7 }]`, one name to a pair. */
|
||||
export function parseCounted(raw: unknown, limit = 10): AdguardCounted[] {
|
||||
const out: AdguardCounted[] = [];
|
||||
for (const pair of arr(raw)) {
|
||||
for (const [name, count] of Object.entries(obj(pair))) {
|
||||
const n = num(count);
|
||||
if (n !== null) out.push({ name, count: n });
|
||||
}
|
||||
}
|
||||
return out.sort((a, b) => b.count - a.count).slice(0, limit);
|
||||
}
|
||||
|
||||
/** Pure: `/control/stats`. */
|
||||
export function parseStats(raw: unknown): AdguardStats | null {
|
||||
const s = obj(raw);
|
||||
const queries = num(s.num_dns_queries);
|
||||
if (queries === null) return null;
|
||||
const blocked = num(s.num_blocked_filtering) ?? 0;
|
||||
const avg = num(s.avg_processing_time);
|
||||
const units = str(s.time_units);
|
||||
const slots = arr(s.dns_queries).length;
|
||||
return {
|
||||
queries,
|
||||
blocked,
|
||||
blockedPercent: queries > 0 ? Math.round((blocked / queries) * 1000) / 10 : null,
|
||||
avgMs: avg === null ? null : Math.round(avg * 10000) / 10,
|
||||
safeBrowsing: num(s.num_replaced_safebrowsing) ?? 0,
|
||||
parental: num(s.num_replaced_parental) ?? 0,
|
||||
safeSearch: num(s.num_replaced_safesearch) ?? 0,
|
||||
periodHours: units === "hours" && slots > 0 ? slots : units === "days" && slots > 0 ? slots * 24 : null,
|
||||
topBlocked: parseCounted(s.top_blocked_domains),
|
||||
topClients: parseCounted(s.top_clients),
|
||||
topQueried: parseCounted(s.top_queried_domains),
|
||||
};
|
||||
}
|
||||
|
||||
/** Pure: `/control/filtering/status` — the lists, and whether filtering is on at all. AdGuard writes "never updated" as the year 1. */
|
||||
export function parseFilters(raw: unknown): { enabled: boolean; lists: AdguardFilterList[] } | null {
|
||||
const f = obj(raw);
|
||||
if (!("filters" in f) && !("enabled" in f)) return null;
|
||||
const lists: AdguardFilterList[] = [];
|
||||
for (const item of arr(f.filters)) {
|
||||
const l = obj(item);
|
||||
const name = str(l.name) ?? str(l.url);
|
||||
if (!name) continue;
|
||||
const updated = str(l.last_updated);
|
||||
const t = updated === null ? NaN : Date.parse(updated);
|
||||
lists.push({ name, enabled: l.enabled !== false, rules: num(l.rules_count), updated: Number.isFinite(t) && t > Date.UTC(2000, 0, 1) ? new Date(t).toISOString() : null });
|
||||
}
|
||||
return { enabled: f.enabled !== false, lists };
|
||||
}
|
||||
|
||||
export function createAdguardAdapter(config: AdguardConfig): AdguardAdapter {
|
||||
const raw = String(config.url ?? "").trim().replace(/\/+$/, "");
|
||||
const base = raw;
|
||||
const insecure = config.insecure === true;
|
||||
const headers: Record<string, string> = { Accept: "application/json" };
|
||||
if (config.username) headers.Authorization = `Basic ${Buffer.from(`${config.username}:${config.password ?? ""}`).toString("base64")}`;
|
||||
|
||||
async function get(path: string): Promise<unknown> {
|
||||
const res = await requestJson(`${base}${path}`, { headers, insecure });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`AdGuard Home did not accept the sign-in (HTTP ${res.status}) — check the user name and password`);
|
||||
if (res.status === 404) throw new Error("AdGuard Home has no API at this address (HTTP 404) — check the address (its own, not a proxy's)");
|
||||
if (res.status < 200 || res.status >= 300) throw new Error(`AdGuard Home error: HTTP ${res.status}`);
|
||||
if (res.json === null) throw new Error("AdGuard Home answered with something that isn't JSON — is the address its own?");
|
||||
return res.json;
|
||||
}
|
||||
|
||||
const soft = async <T>(run: () => Promise<T>): Promise<T | null> => {
|
||||
try {
|
||||
return await run();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
function dnsHostOf(): string {
|
||||
const given = str(config.dnsHost);
|
||||
if (given) return given.replace(/^\[|\]$/g, "");
|
||||
try {
|
||||
return new URL(raw).hostname.replace(/^\[|\]$/g, "");
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
async function read(): Promise<AdguardRead> {
|
||||
// The status must work: it says it is there, whether it runs and whether it protects.
|
||||
const status = obj(await get("/control/status"));
|
||||
const dnsPort = num(status.dns_port) ?? 53;
|
||||
const name = str(config.probeDomain) ?? DEFAULT_PROBE_DOMAIN;
|
||||
const host = dnsHostOf();
|
||||
const [stats, filters, probe] = await Promise.all([soft(async () => parseStats(await get("/control/stats"))), soft(async () => parseFilters(await get("/control/filtering/status"))), host ? probeDns(host, dnsPort, name) : Promise.resolve({ ok: false, ms: null, error: "no address to ask" } as ProbeResult)]);
|
||||
return { version: str(status.version), running: status.running !== false, protectionEnabled: status.protection_enabled !== false, dnsPort, stats, filters, probe: { ...probe, name, host } };
|
||||
}
|
||||
|
||||
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
await get("/control/status");
|
||||
return { ok: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
return withDiagLogging("adguard", { ping, read });
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* Asks a DNS server a plain question over UDP — "what is the address of example.com?" — and says whether it answered. That is the only honest test of "DNS is working": AdGuard Home's own API
|
||||
* can be up while its DNS server is not (or its upstreams are not answering), and what a household notices is a name that does not resolve. Nothing is read from the answer but whether there is
|
||||
* one and how it ended: an "address not found" (NXDOMAIN) is an answer too — the resolver works — while a server failure (SERVFAIL, REFUSED …) is not.
|
||||
*/
|
||||
import * as dgram from "node:dgram";
|
||||
import * as net from "node:net";
|
||||
import { randomInt } from "node:crypto";
|
||||
|
||||
export const PROBE_TIMEOUT_MS = 3000;
|
||||
|
||||
/** Pure: a standard recursive query for the address (A record) of a name. */
|
||||
export function buildQuery(name: string, id: number): Buffer {
|
||||
const labels = name.replace(/\.$/, "").split(".").filter((l) => l !== "");
|
||||
if (labels.length === 0 || labels.some((l) => l.length > 63 || !/^[A-Za-z0-9_-]+$/.test(l))) throw new Error(`"${name}" is not a name to ask about`);
|
||||
const header = Buffer.alloc(12);
|
||||
header.writeUInt16BE(id & 0xffff, 0);
|
||||
header.writeUInt16BE(0x0100, 2); // recursion desired
|
||||
header.writeUInt16BE(1, 4); // one question
|
||||
const parts = labels.map((l) => Buffer.concat([Buffer.from([l.length]), Buffer.from(l, "ascii")]));
|
||||
const tail = Buffer.alloc(4);
|
||||
tail.writeUInt16BE(1, 0); // A
|
||||
tail.writeUInt16BE(1, 2); // IN
|
||||
return Buffer.concat([header, ...parts, Buffer.from([0]), tail]);
|
||||
}
|
||||
|
||||
const RCODES = ["no error", "format error", "server failure (SERVFAIL)", "no such name (NXDOMAIN)", "not implemented", "refused (REFUSED)"];
|
||||
|
||||
/** Pure: whether a packet is the answer to the query with this id, and how it ended; null when it is not an answer to it. */
|
||||
export function parseResponse(packet: Buffer, id: number): { rcode: number; answers: number } | null {
|
||||
if (packet.length < 12) return null;
|
||||
if (packet.readUInt16BE(0) !== (id & 0xffff)) return null;
|
||||
const flags = packet.readUInt16BE(2);
|
||||
if ((flags & 0x8000) === 0) return null; // a query, not a response
|
||||
return { rcode: flags & 0x000f, answers: packet.readUInt16BE(6) };
|
||||
}
|
||||
|
||||
export interface ProbeResult {
|
||||
ok: boolean;
|
||||
/** How long the answer took, in milliseconds; null when there was none. */
|
||||
ms: number | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
/** Whether a DNS server answers a question for a name. Never throws. */
|
||||
export function probeDns(host: string, port: number, name: string, timeoutMs: number = PROBE_TIMEOUT_MS): Promise<ProbeResult> {
|
||||
return new Promise((resolve) => {
|
||||
let query: Buffer;
|
||||
const id = randomInt(0, 0x10000);
|
||||
try {
|
||||
query = buildQuery(name, id);
|
||||
} catch (err) {
|
||||
return resolve({ ok: false, ms: null, error: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
const socket = dgram.createSocket(net.isIPv6(host) ? "udp6" : "udp4");
|
||||
const started = Date.now();
|
||||
let done = false;
|
||||
const finish = (result: ProbeResult) => {
|
||||
if (done) return;
|
||||
done = true;
|
||||
clearTimeout(timer);
|
||||
try {
|
||||
socket.close();
|
||||
} catch {
|
||||
// already closed
|
||||
}
|
||||
resolve(result);
|
||||
};
|
||||
const timer = setTimeout(() => finish({ ok: false, ms: null, error: `no answer in ${Math.round(timeoutMs / 1000)} s` }), timeoutMs);
|
||||
socket.on("error", (err) => finish({ ok: false, ms: null, error: err.message }));
|
||||
socket.on("message", (packet) => {
|
||||
const r = parseResponse(packet, id);
|
||||
if (!r) return; // not ours: keep waiting
|
||||
const ms = Date.now() - started;
|
||||
// An answer, or "no such name", means the resolver works; any other ending means it is up but cannot resolve.
|
||||
if (r.rcode === 0 || r.rcode === 3) finish({ ok: true, ms, error: null });
|
||||
else finish({ ok: false, ms, error: `answered, but with ${RCODES[r.rcode] ?? `error ${r.rcode}`}` });
|
||||
});
|
||||
socket.send(query, port, host, (err) => {
|
||||
if (err) finish({ ok: false, ms: null, error: err.message });
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -122,6 +122,23 @@ export interface DockhandEnvironmentVulns {
|
||||
critical: DockhandVulnFinding[];
|
||||
}
|
||||
|
||||
/** An image a running container uses, and when that image was built (Docker's own "Created"). */
|
||||
export interface DockhandRunningImage {
|
||||
/** As the container names it ("nginx:1.25"). */
|
||||
image: string;
|
||||
/** ISO; null when the image could not be found in the list (an untagged one, a name that was typed differently). */
|
||||
created: string | null;
|
||||
/** The running containers that use it. */
|
||||
containers: string[];
|
||||
}
|
||||
|
||||
export interface DockhandEnvironmentImages {
|
||||
environmentId: number;
|
||||
environmentName: string;
|
||||
/** Null when the images or the containers of this environment could not be read — which is not the same as none. */
|
||||
images: DockhandRunningImage[] | null;
|
||||
}
|
||||
|
||||
export interface DockhandVulnerabilityRead {
|
||||
environments: DockhandEnvironmentVulns[];
|
||||
}
|
||||
@@ -141,6 +158,56 @@ export function parseVulnSummary(raw: unknown): DockhandVulnSummary | null {
|
||||
return { critical, high: asCount(s.high) ?? 0, medium: asCount(s.medium) ?? 0, low: asCount(s.low) ?? 0, imagesScanned: asCount(s.imagesScanned), totalImages: asCount(s.totalImages) };
|
||||
}
|
||||
|
||||
/** Pure: when an image was built, from Docker's "Created" — seconds since 1970 (what Docker's list gives), milliseconds, or a date; null for anything else, and for Docker's "never" (0). */
|
||||
export function parseImageCreated(v: unknown): string | null {
|
||||
if (typeof v === "number" && Number.isFinite(v) && v > 0) {
|
||||
const ms = v < 1e11 ? v * 1000 : v;
|
||||
return new Date(ms).toISOString();
|
||||
}
|
||||
if (typeof v === "string" && v.trim() !== "") {
|
||||
const t = Date.parse(v);
|
||||
if (Number.isFinite(t) && t > 0) return new Date(t).toISOString();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Pure: the images of /api/images — Docker's own list: `Id`, `RepoTags` and `Created` (a lower-case spelling is read as well). */
|
||||
export function parseImageList(raw: unknown): { id: string; tags: string[]; created: string | null }[] {
|
||||
const out: { id: string; tags: string[]; created: string | null }[] = [];
|
||||
for (const item of Array.isArray(raw) ? raw : []) {
|
||||
const i = (item !== null && typeof item === "object" ? item : {}) as Record<string, unknown>;
|
||||
const id = asStr(i.Id) ?? asStr(i.id);
|
||||
if (!id) continue;
|
||||
const tags = (Array.isArray(i.RepoTags) ? i.RepoTags : Array.isArray(i.repoTags) ? i.repoTags : []).map(asStr).filter((t): t is string => t !== null && t !== "<none>:<none>");
|
||||
out.push({ id, tags, created: parseImageCreated(i.Created ?? i.created) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** An image name as Docker lists it: without the registry Docker Hub is by default, and with a tag. */
|
||||
const canonicalImage = (name: string): string => {
|
||||
const bare = name.replace(/^(?:docker\.io\/)?(?:library\/)?/, "");
|
||||
const last = bare.split("/").pop() ?? bare;
|
||||
return last.includes(":") || bare.includes("@") ? bare : `${bare}:latest`;
|
||||
};
|
||||
|
||||
/** Pure: the images the running containers use, each with when it was built — matched by the name the container gives (tag), or by id when it gives one. */
|
||||
export function matchRunningImages(containers: { name: string; image: string; state: string }[], images: { id: string; tags: string[]; created: string | null }[]): DockhandRunningImage[] {
|
||||
const byTag = new Map<string, string | null>();
|
||||
for (const i of images) for (const t of i.tags) byTag.set(canonicalImage(t), i.created);
|
||||
const byId = new Map(images.map((i) => [i.id.replace(/^sha256:/, ""), i.created]));
|
||||
const out = new Map<string, DockhandRunningImage>();
|
||||
for (const c of containers) {
|
||||
if (c.state !== "running" || !c.image) continue;
|
||||
const sha = c.image.replace(/^sha256:/, "");
|
||||
const created = byTag.get(canonicalImage(c.image)) ?? (/^[0-9a-f]{12,64}$/.test(sha) ? ([...byId.entries()].find(([id]) => id.startsWith(sha))?.[1] ?? null) : null);
|
||||
const entry = out.get(c.image) ?? { image: c.image, created: created ?? null, containers: [] };
|
||||
entry.containers.push(c.name);
|
||||
out.set(c.image, entry);
|
||||
}
|
||||
return [...out.values()].sort((a, b) => a.image.localeCompare(b.image));
|
||||
}
|
||||
|
||||
/** Pure: the findings of one page of /api/vulnerabilities — each `{ cve, package, severity, installedVersion, fixedVersion, imageName, containers: [{ id, name }] }`. */
|
||||
export function parseVulnFindings(raw: unknown): DockhandVulnFinding[] {
|
||||
const list = asObj(raw).findings;
|
||||
@@ -160,6 +227,8 @@ export interface DockhandAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
/** What Dockhand's own stored scans say, per environment. Reads only: no scan is started. */
|
||||
readVulnerabilities(): Promise<DockhandVulnerabilityRead>;
|
||||
/** For each Docker host, the images its running containers use and when each was built (GET /api/images and /api/containers). Reads only. */
|
||||
readImages(): Promise<{ environments: DockhandEnvironmentImages[] }>;
|
||||
listContainers(): Promise<DockhandContainer[]>;
|
||||
listContainersDetailed(): Promise<DockhandContainerRead>;
|
||||
startContainer(environmentId: number, containerId: string): Promise<void>;
|
||||
@@ -269,6 +338,23 @@ export function createDockhandAdapter(config: DockhandConfig): DockhandAdapter {
|
||||
return { environments: perEnv };
|
||||
}
|
||||
|
||||
async function readImages(): Promise<{ environments: DockhandEnvironmentImages[] }> {
|
||||
const environments = await listEnvironments();
|
||||
const perEnv = await Promise.all(
|
||||
environments.map(async (env): Promise<DockhandEnvironmentImages> => {
|
||||
const known = { environmentId: env.id, environmentName: env.name };
|
||||
try {
|
||||
const [list, containers] = await Promise.all([api("GET", `/api/images?env=${numericSegment(env.id, "environment ID")}`), api("GET", `/api/containers?env=${numericSegment(env.id, "environment ID")}&all=true`)]);
|
||||
const running = (Array.isArray(containers) ? containers : []).map((c: any) => ({ name: String(c.name ?? c.id ?? ""), image: String(c.image ?? ""), state: String(c.state ?? "") }));
|
||||
return { ...known, images: matchRunningImages(running, parseImageList(list)) };
|
||||
} catch {
|
||||
return { ...known, images: null };
|
||||
}
|
||||
}),
|
||||
);
|
||||
return { environments: perEnv };
|
||||
}
|
||||
|
||||
async function listContainers(): Promise<DockhandContainer[]> {
|
||||
return (await listContainersDetailed()).containers;
|
||||
}
|
||||
@@ -314,5 +400,5 @@ export function createDockhandAdapter(config: DockhandConfig): DockhandAdapter {
|
||||
}
|
||||
}
|
||||
|
||||
return withDiagLogging("dockhand", { ping, readVulnerabilities, listContainers, listContainersDetailed, startContainer, stopContainer, restartContainer, checkForUpdates });
|
||||
return withDiagLogging("dockhand", { ping, readVulnerabilities, readImages, listContainers, listContainersDetailed, startContainer, stopContainer, restartContainer, checkForUpdates });
|
||||
}
|
||||
@@ -129,6 +129,19 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
|
||||
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "Settings → General → Security → API Key" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
adguard: [
|
||||
{ key: "url", label: "AdGuard Home URL", secret: false, placeholder: "http://adguard.example.lan:3000" },
|
||||
{ key: "username", label: "User name (optional)", secret: false, optional: true, placeholder: "empty when AdGuard Home has no sign-in" },
|
||||
{ key: "password", label: "Password (optional)", secret: true, optional: true, type: "password" },
|
||||
{ key: "dnsHost", label: "DNS server address (optional)", secret: false, optional: true, placeholder: "the host of the URL — only if its DNS server is somewhere else" },
|
||||
{ key: "probeDomain", label: "Name to test DNS with (optional)", secret: false, optional: true, placeholder: "example.com" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
grafana: [
|
||||
{ key: "url", label: "Grafana URL", secret: false, placeholder: "http://grafana.example.lan:3000" },
|
||||
{ key: "token", label: "Service account token", secret: true, type: "password", placeholder: "Administration → Service accounts → add a token (role Viewer)" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
fragnet: [
|
||||
{ key: "token", label: "API key", secret: true, type: "password", placeholder: "ptlc_… — the game panel: profile → API Credentials (not an Application key, ptla_)" },
|
||||
{ key: "url", label: "Panel address (optional)", secret: false, optional: true, placeholder: "https://panel.fragnet.net — only if yours is somewhere else" },
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
/**
|
||||
* Grafana adapter. Config: url and token (a service account's token — Administration → Users and access → Service accounts → add a token; the **Viewer** role is enough for the rules, and to also
|
||||
* test the data sources the account needs permission to read and query them) and optionally insecure.
|
||||
*
|
||||
* Reading only: the alert rules and what state each is in (`GET /api/prometheus/grafana/api/v1/rules`, the Prometheus-compatible view of Grafana-managed alerting: firing, pending, and whether it
|
||||
* could be evaluated), and the data sources with the result of Grafana's own health check of each (`GET /api/datasources`, `GET /api/datasources/uid/{uid}/health`). Nothing is changed: no rule is
|
||||
* silenced or paused, no data source edited.
|
||||
*
|
||||
* Endpoints and fields are from Grafana's HTTP API documentation and have not been checked against a live Grafana here; every field is read defensively. A data source whose type has no health
|
||||
* check is "not tested" (null), which is not the same as healthy.
|
||||
*/
|
||||
import { requestJson } from "../httpJson.js";
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
|
||||
export interface GrafanaConfig {
|
||||
url: string;
|
||||
token: string;
|
||||
insecure?: boolean;
|
||||
}
|
||||
|
||||
export interface GrafanaRule {
|
||||
name: string;
|
||||
uid: string | null;
|
||||
group: string;
|
||||
/** The folder it is in (Grafana's "file"). */
|
||||
folder: string | null;
|
||||
/** inactive, pending or firing. */
|
||||
state: string;
|
||||
/** ok, error or nodata. */
|
||||
health: string;
|
||||
lastError: string | null;
|
||||
/** The rule's own `severity` label, lower case, when it has one. */
|
||||
severity: string | null;
|
||||
summary: string | null;
|
||||
/** ISO: when the oldest of its alerts started. */
|
||||
activeAt: string | null;
|
||||
}
|
||||
|
||||
export interface GrafanaDatasource {
|
||||
uid: string;
|
||||
name: string;
|
||||
type: string;
|
||||
isDefault: boolean;
|
||||
/** The result of Grafana's health check; null when the type has none or it could not be asked. */
|
||||
ok: boolean | null;
|
||||
message: string | null;
|
||||
}
|
||||
|
||||
export interface GrafanaRead {
|
||||
version: string | null;
|
||||
/** The rules that are firing, pending or cannot be evaluated, and how many rules there are in all; null when the rules could not be read. */
|
||||
rules: { total: number; items: GrafanaRule[] } | null;
|
||||
/** Null when the data sources could not be listed. */
|
||||
datasources: GrafanaDatasource[] | null;
|
||||
}
|
||||
|
||||
export interface GrafanaAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(): Promise<GrafanaRead>;
|
||||
}
|
||||
|
||||
const MAX_DATASOURCES = 40;
|
||||
const MAX_ITEMS = 200;
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
|
||||
const arr = (v: unknown): unknown[] => (Array.isArray(v) ? v : []);
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
|
||||
const iso = (v: unknown): string | null => {
|
||||
const s = str(v);
|
||||
const t = s === null ? NaN : Date.parse(s);
|
||||
// Grafana writes "no time" as the year 1.
|
||||
return Number.isFinite(t) && t > Date.UTC(2000, 0, 1) ? new Date(t).toISOString() : null;
|
||||
};
|
||||
|
||||
/** Pure: the rules of `/api/prometheus/grafana/api/v1/rules` — every rule counted, those that are not quietly fine kept. */
|
||||
export function parseRules(raw: unknown): { total: number; items: GrafanaRule[] } {
|
||||
let total = 0;
|
||||
const items: GrafanaRule[] = [];
|
||||
for (const g of arr(obj(obj(raw).data).groups)) {
|
||||
const group = obj(g);
|
||||
const groupName = str(group.name) ?? "";
|
||||
for (const r of arr(group.rules)) {
|
||||
const rule = obj(r);
|
||||
const name = str(rule.name);
|
||||
if (!name) continue;
|
||||
total++;
|
||||
const state = (str(rule.state) ?? "inactive").toLowerCase();
|
||||
const health = (str(rule.health) ?? "ok").toLowerCase();
|
||||
if (state === "inactive" && health !== "error") continue;
|
||||
if (state === "normal" && health !== "error") continue;
|
||||
const labels = obj(rule.labels);
|
||||
const annotations = obj(rule.annotations);
|
||||
const activeAt = arr(rule.alerts)
|
||||
.map((a) => iso(obj(a).activeAt))
|
||||
.filter((t): t is string => t !== null)
|
||||
.sort()[0] ?? null;
|
||||
items.push({
|
||||
name,
|
||||
uid: str(rule.uid),
|
||||
group: groupName,
|
||||
folder: str(group.file),
|
||||
state: state === "alerting" ? "firing" : state,
|
||||
health,
|
||||
lastError: str(rule.lastError),
|
||||
severity: str(labels.severity)?.toLowerCase() ?? null,
|
||||
summary: str(annotations.summary) ?? str(annotations.description),
|
||||
activeAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
return { total, items: items.slice(0, MAX_ITEMS) };
|
||||
}
|
||||
|
||||
/** Pure: the data sources of `/api/datasources` (health not yet asked). */
|
||||
export function parseDatasources(raw: unknown): Omit<GrafanaDatasource, "ok" | "message">[] {
|
||||
const out: Omit<GrafanaDatasource, "ok" | "message">[] = [];
|
||||
for (const d of arr(raw)) {
|
||||
const o = obj(d);
|
||||
const uid = str(o.uid);
|
||||
const name = str(o.name);
|
||||
if (!uid || !name) continue;
|
||||
out.push({ uid, name, type: str(o.type) ?? "unknown", isDefault: o.isDefault === true });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: Grafana's health answer for one data source — `{ status: "OK" | "ERROR", message }`; anything else is "not tested". */
|
||||
export function parseDatasourceHealth(raw: unknown): { ok: boolean | null; message: string | null } {
|
||||
const h = obj(raw);
|
||||
const status = str(h.status)?.toUpperCase();
|
||||
if (status === "OK") return { ok: true, message: str(h.message) };
|
||||
if (status === "ERROR") return { ok: false, message: str(h.message) };
|
||||
return { ok: null, message: null };
|
||||
}
|
||||
|
||||
export function createGrafanaAdapter(config: GrafanaConfig): GrafanaAdapter {
|
||||
const base = String(config.url ?? "").trim().replace(/\/+$/, "");
|
||||
const insecure = config.insecure === true;
|
||||
const headers = { Authorization: `Bearer ${String(config.token ?? "").trim()}`, Accept: "application/json" };
|
||||
|
||||
async function get(path: string): Promise<{ status: number; json: unknown }> {
|
||||
const res = await requestJson(`${base}${path}`, { headers, insecure });
|
||||
if (res.status === 401) throw new Error("Grafana did not accept the token (HTTP 401) — make a new service account token, and check it has not expired or been deleted");
|
||||
return { status: res.status, json: res.json };
|
||||
}
|
||||
|
||||
async function must(path: string): Promise<unknown> {
|
||||
const res = await get(path);
|
||||
if (res.status === 403) throw new Error(`Grafana says this token may not read ${path.split("?")[0]} (HTTP 403) — give its service account the Viewer role`);
|
||||
if (res.status === 404) throw new Error("Grafana has no API at this address (HTTP 404) — check the address (its own, not a proxy's)");
|
||||
if (res.status < 200 || res.status >= 300) throw new Error(`Grafana error: HTTP ${res.status}`);
|
||||
if (res.json === null) throw new Error("Grafana answered with something that isn't JSON — is the address its own?");
|
||||
return res.json;
|
||||
}
|
||||
|
||||
const soft = async <T>(run: () => Promise<T>): Promise<T | null> => {
|
||||
try {
|
||||
return await run();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
async function read(): Promise<GrafanaRead> {
|
||||
// The health must work: it says it is there (and the token is tried on what needs it, below).
|
||||
const health = obj(await must("/api/health"));
|
||||
const [rules, datasources] = await Promise.all([
|
||||
soft(async () => parseRules(await must("/api/prometheus/grafana/api/v1/rules"))),
|
||||
soft(async () => {
|
||||
const list = parseDatasources(await must("/api/datasources")).slice(0, MAX_DATASOURCES);
|
||||
return Promise.all(
|
||||
list.map(async (d): Promise<GrafanaDatasource> => {
|
||||
const h = await soft(async () => {
|
||||
const res = await get(`/api/datasources/uid/${encodeURIComponent(d.uid)}/health`);
|
||||
return res.status >= 200 && res.status < 500 ? parseDatasourceHealth(res.json) : { ok: null, message: null };
|
||||
});
|
||||
return { ...d, ...(h ?? { ok: null, message: null }) };
|
||||
}),
|
||||
);
|
||||
}),
|
||||
]);
|
||||
if (rules === null && datasources === null) throw new Error("Grafana's health answered, but neither the rules nor the data sources could be read — check the token's permissions");
|
||||
return { version: str(health.version), rules, datasources };
|
||||
}
|
||||
|
||||
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
await must("/api/health");
|
||||
await must("/api/org");
|
||||
return { ok: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
return withDiagLogging("grafana", { ping, read });
|
||||
}
|
||||
@@ -25,6 +25,8 @@ import { createHetznerAdapter } from "./hetzner/adapter.js";
|
||||
import { createSpeedtestAdapter } from "./speedtest/adapter.js";
|
||||
import { createFragnetAdapter } from "./fragnet/adapter.js";
|
||||
import { createArrAdapter } from "./arr/adapter.js";
|
||||
import { createAdguardAdapter } from "./adguard/adapter.js";
|
||||
import { createGrafanaAdapter } from "./grafana/adapter.js";
|
||||
import { createNextcloudAdapter } from "./nextcloud/adapter.js";
|
||||
|
||||
export interface PingableAdapter {
|
||||
@@ -90,6 +92,10 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
|
||||
case "radarr":
|
||||
case "prowlarr":
|
||||
return createArrAdapter(type, config as any);
|
||||
case "adguard":
|
||||
return createAdguardAdapter(config as any);
|
||||
case "grafana":
|
||||
return createGrafanaAdapter(config as any);
|
||||
case "netbox":
|
||||
return createNetBoxAdapter(config as any);
|
||||
case "nextcloud":
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { adguardState, integrations } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { adguardProblems, adguardSyncStatuses, isAdguardFresh, staleLists, storedOf } from "../services/adguardSync.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const adguardRouter = Router();
|
||||
adguardRouter.use(requireAuth);
|
||||
|
||||
/** Every AdGuard Home, how it is, from what the last reads stored: whether it runs and protects, whether DNS answers, the statistics, its lists, and what is wrong. Everyone signed in can see it. */
|
||||
adguardRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, states, statuses] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled, baseUrl: integrations.baseUrl }).from(integrations).where(eq(integrations.type, "adguard")).orderBy(integrations.name),
|
||||
db.select().from(adguardState),
|
||||
adguardSyncStatuses(),
|
||||
]);
|
||||
const byId = new Map(states.map((s) => [s.integrationId, s]));
|
||||
const now = Date.now();
|
||||
res.json({
|
||||
instances: rows.map((r) => {
|
||||
const row = byId.get(r.id);
|
||||
const state = row ? storedOf(row) : null;
|
||||
const status = statuses[r.id] ?? null;
|
||||
const fresh = isAdguardFresh(status ?? undefined, now);
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
enabled: r.enabled,
|
||||
url: r.baseUrl,
|
||||
status,
|
||||
fresh,
|
||||
state: state && { ...state, staleLists: staleLists(state.filters, now).map((l) => l.name) },
|
||||
problems: state && fresh ? adguardProblems(r.id, r.name, state, now) : [],
|
||||
};
|
||||
}),
|
||||
});
|
||||
}));
|
||||
@@ -0,0 +1,30 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { grafanaState, integrations } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { grafanaProblems, grafanaSyncStatuses, isGrafanaFresh, storedOf } from "../services/grafanaSync.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const grafanaRouter = Router();
|
||||
grafanaRouter.use(requireAuth);
|
||||
|
||||
/** Every Grafana, how it is, from what the last reads stored: which rules are firing or cannot be evaluated, how its data sources test, and what is wrong. Everyone signed in can see it. */
|
||||
grafanaRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, states, statuses] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled, baseUrl: integrations.baseUrl }).from(integrations).where(eq(integrations.type, "grafana")).orderBy(integrations.name),
|
||||
db.select().from(grafanaState),
|
||||
grafanaSyncStatuses(),
|
||||
]);
|
||||
const byId = new Map(states.map((s) => [s.integrationId, s]));
|
||||
const now = Date.now();
|
||||
res.json({
|
||||
instances: rows.map((r) => {
|
||||
const row = byId.get(r.id);
|
||||
const state = row ? storedOf(row) : null;
|
||||
const status = statuses[r.id] ?? null;
|
||||
const fresh = isGrafanaFresh(status ?? undefined, now);
|
||||
return { id: r.id, name: r.name, enabled: r.enabled, url: r.baseUrl, status, fresh, state, problems: state && fresh ? grafanaProblems(r.id, r.name, state, now) : [] };
|
||||
}),
|
||||
});
|
||||
}));
|
||||
@@ -40,8 +40,10 @@ import { forgetHetzner, syncHetzner } from "../services/hetznerSync.js";
|
||||
import { forgetSpeedtest, syncSpeedtest } from "../services/speedtestSync.js";
|
||||
import { forgetFragnet, syncFragnet } from "../services/fragnetSync.js";
|
||||
import { forgetArr, syncArr } from "../services/arrSync.js";
|
||||
import { forgetAdguard, syncAdguard } from "../services/adguardSync.js";
|
||||
import { forgetGrafana, syncGrafana } from "../services/grafanaSync.js";
|
||||
import { forgetNextcloud, syncNextcloud } from "../services/nextcloudSync.js";
|
||||
import { dockerVulnSyncStatuses, findingsOf, forgetDockerVulns, isDockerVulnFresh, syncDockerVulns } from "../services/dockerVulnSync.js";
|
||||
import { dockerVulnSyncStatuses, findingsOf, forgetDockerVulns, imageAgeDays, imagesOf, isDockerVulnFresh, syncDockerVulns } from "../services/dockerVulnSync.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
@@ -155,6 +157,8 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
|
||||
if (type === "speedtest") void syncSpeedtest(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "fragnet") void syncFragnet(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "sonarr" || type === "radarr" || type === "prowlarr") void syncArr(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "adguard") void syncAdguard(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "grafana") void syncGrafana(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "nextcloud") void syncNextcloud(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "dockhand") void syncDockerVulns(created.id).then(() => invalidateAlertsCache());
|
||||
|
||||
@@ -271,6 +275,8 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
if (updated.type === "speedtest" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncSpeedtest(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "fragnet" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncFragnet(updated.id).then(() => invalidateAlertsCache());
|
||||
if ((updated.type === "sonarr" || updated.type === "radarr" || updated.type === "prowlarr") && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncArr(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "adguard" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncAdguard(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "grafana" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncGrafana(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "nextcloud" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncNextcloud(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "dockhand" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncDockerVulns(updated.id).then(() => invalidateAlertsCache());
|
||||
|
||||
@@ -363,6 +369,8 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
if (existing.type === "speedtest") await forgetSpeedtest(id);
|
||||
if (existing.type === "fragnet") await forgetFragnet(id);
|
||||
if (existing.type === "sonarr" || existing.type === "radarr" || existing.type === "prowlarr") await forgetArr(id);
|
||||
if (existing.type === "adguard") await forgetAdguard(id);
|
||||
if (existing.type === "grafana") await forgetGrafana(id);
|
||||
if (existing.type === "nextcloud") await forgetNextcloud(id);
|
||||
if (existing.type === "dockhand") await forgetDockerVulns(id);
|
||||
await db.delete(integrations).where(eq(integrations.id, id));
|
||||
@@ -595,6 +603,15 @@ async function requireDockhandAdapter(req: Request, res: Response) {
|
||||
return { integration: loaded.integration, adapter: createDockhandAdapter(loaded.config as any) };
|
||||
}
|
||||
|
||||
/** The running images of a Docker host by how old they are (the oldest five), and how many are older than the threshold; null when the images were never read. */
|
||||
function imageAges(row: Parameters<typeof imagesOf>[0] & { imagesAt: string | null }, thresholdDays: number) {
|
||||
const images = imagesOf(row);
|
||||
if (images === null) return null;
|
||||
const now = Date.now();
|
||||
const dated = images.filter((i): i is typeof i & { created: string } => i.created !== null).map((i) => ({ image: i.image, ageDays: imageAgeDays(i.created, now), containers: i.containers })).sort((a, b) => b.ageDays - a.ageDays || a.image.localeCompare(b.image));
|
||||
return { at: row.imagesAt, running: images.length, undated: images.length - dated.length, over: thresholdDays > 0 ? dated.filter((i) => i.ageDays > thresholdDays).length : null, oldest: dated.slice(0, 5) };
|
||||
}
|
||||
|
||||
// What Dockhand's own vulnerability scans add up to for each of its Docker hosts, as last read (nothing is asked of Dockhand here, and nothing is scanned).
|
||||
integrationsRouter.get("/:id/dockhand/vulnerabilities", asyncHandler(async (req, res) => {
|
||||
const id = Number(req.params.id);
|
||||
@@ -606,11 +623,12 @@ integrationsRouter.get("/:id/dockhand/vulnerabilities", asyncHandler(async (req,
|
||||
res.json({
|
||||
alerts: settings.healthChecks.vulnerabilityAlerts,
|
||||
highThreshold: settings.healthChecks.vulnerabilityHighThreshold ?? 0,
|
||||
imageAgeDays: settings.healthChecks.imageAgeDays ?? 0,
|
||||
status,
|
||||
fresh: isDockerVulnFresh(status ?? undefined, Date.now()),
|
||||
environments: rows
|
||||
.sort((a, b) => a.environmentName.localeCompare(b.environmentName))
|
||||
.map((r) => ({ environmentId: r.environmentId, environmentName: r.environmentName, critical: r.critical, high: r.high, medium: r.medium, low: r.low, imagesScanned: r.imagesScanned, totalImages: r.totalImages, findings: findingsOf(r), updatedAt: r.updatedAt })),
|
||||
.map((r) => ({ environmentId: r.environmentId, environmentName: r.environmentName, critical: r.critical, high: r.high, medium: r.medium, low: r.low, imagesScanned: r.imagesScanned, totalImages: r.totalImages, findings: findingsOf(r), images: imageAges(r, settings.healthChecks.imageAgeDays ?? 0), updatedAt: r.updatedAt })),
|
||||
});
|
||||
}));
|
||||
|
||||
|
||||
+39
-2
@@ -40,9 +40,9 @@ export function parseDashboardLayout(stored: string | null): DashboardLayout {
|
||||
|
||||
meRouter.get("/", requireAuth, asyncHandler(async (req, res) => {
|
||||
const me = req.currentUser!;
|
||||
const { id, email, name, role, oidcSub, favorites, dashboardLayout } = me;
|
||||
const { id, email, name, role, oidcSub, favorites, dashboardLayout, serverPageLayout } = me;
|
||||
const avatarUrl = avatarUrlFor(me, (await getSettings()).profilePictures);
|
||||
res.json({ user: { id, sub: oidcSub, email, name, role, avatarUrl, favorites: parseFavorites(favorites), dashboard: parseDashboardLayout(dashboardLayout) } });
|
||||
res.json({ user: { id, sub: oidcSub, email, name, role, avatarUrl, favorites: parseFavorites(favorites), dashboard: parseDashboardLayout(dashboardLayout), serverPage: parsePageLayout(serverPageLayout) } });
|
||||
}));
|
||||
|
||||
// Paths inside the app only: a slash, then letters, digits, dashes, underscores and further slashes — and, for a saved view of a page,
|
||||
@@ -68,6 +68,31 @@ const dashboardSchema = z.object({
|
||||
order: z.record(cardName, z.number().int().min(0).max(1000)).refine((o) => Object.keys(o).length <= MAX_LAYOUT_CARDS, "Too many cards"),
|
||||
});
|
||||
|
||||
export interface PageLayout {
|
||||
/** Blocks (cards) the person folded: only their header shows. */
|
||||
folded: string[];
|
||||
/** Position of a block among the others on the page; lower comes first. */
|
||||
order: Record<string, number>;
|
||||
}
|
||||
|
||||
const blockName = z.string().regex(/^[a-z0-9-]{1,40}$/);
|
||||
const pageLayoutSchema = z.object({
|
||||
folded: z.array(blockName).max(MAX_LAYOUT_CARDS),
|
||||
order: z.record(blockName, z.number().int().min(0).max(1000)).refine((o) => Object.keys(o).length <= MAX_LAYOUT_CARDS, "Too many blocks"),
|
||||
});
|
||||
|
||||
/** What is stored for a page, read back as a layout. Anything that isn't one is the page as drawn rather than an error. */
|
||||
export function parsePageLayout(stored: string | null): PageLayout {
|
||||
const empty: PageLayout = { folded: [], order: {} };
|
||||
if (!stored) return empty;
|
||||
try {
|
||||
const parsed = pageLayoutSchema.safeParse(JSON.parse(stored));
|
||||
return parsed.success ? parsed.data : empty;
|
||||
} catch {
|
||||
return empty;
|
||||
}
|
||||
}
|
||||
|
||||
// Replaces the signed-in person's pinned pages. Only ever their own.
|
||||
meRouter.put("/favorites", requireAuth, asyncHandler(async (req, res) => {
|
||||
const parsed = favoritesSchema.safeParse(req.body);
|
||||
@@ -104,6 +129,18 @@ meRouter.post("/changes/seen", requireAuth, asyncHandler(async (req, res) => {
|
||||
res.json({ lastLookedAt: at });
|
||||
}));
|
||||
|
||||
// Replaces how the signed-in person arranged a server's overview page. Only ever their own.
|
||||
meRouter.put("/server-page", requireAuth, asyncHandler(async (req, res) => {
|
||||
const parsed = pageLayoutSchema.safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
return res.status(400).json({ error: "invalid_body", message: "Name the blocks to fold and the order of the rest.", details: parsed.error.flatten() });
|
||||
}
|
||||
const layout: PageLayout = { folded: [...new Set(parsed.data.folded)], order: parsed.data.order };
|
||||
const empty = layout.folded.length === 0 && Object.keys(layout.order).length === 0;
|
||||
await db.update(users).set({ serverPageLayout: empty ? null : JSON.stringify(layout) }).where(eq(users.id, req.currentUser!.id));
|
||||
res.json({ serverPage: layout });
|
||||
}));
|
||||
|
||||
// Replaces how the signed-in person arranged the dashboard. Only ever their own.
|
||||
meRouter.put("/dashboard", requireAuth, asyncHandler(async (req, res) => {
|
||||
const parsed = dashboardSchema.safeParse(req.body);
|
||||
|
||||
@@ -144,7 +144,7 @@ const updateSchema = z.object({
|
||||
.partial()
|
||||
.optional(),
|
||||
healthChecks: z
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365), baselineAlerts: z.boolean(), containerAlerts: z.boolean(), vulnerabilityAlerts: z.boolean().default(true), vulnerabilityHighThreshold: z.number().int().min(0).max(10000).default(0), containerIgnore: z.array(z.string().trim().min(1).max(100)).max(200) })
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean(), flapRaises: z.number().int().min(0).max(20), deviceOfflineMinutes: z.number().int().min(5).max(1440), osSupportWarnDays: z.number().int().min(0).max(365), baselineAlerts: z.boolean(), containerAlerts: z.boolean(), vulnerabilityAlerts: z.boolean().default(true), vulnerabilityHighThreshold: z.number().int().min(0).max(10000).default(0), imageAgeDays: z.number().int().min(0).max(3650).default(0), containerIgnore: z.array(z.string().trim().min(1).max(100)).max(200) })
|
||||
.partial()
|
||||
.optional(),
|
||||
quietHours: z
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
/**
|
||||
* Keeps what AdGuard Home says up to date: every two minutes each AdGuard Home integration is read — whether its DNS server runs and protects, the statistics, its filter lists — and one real
|
||||
* DNS question is put to its DNS server; the state is stored. Pages and alerts are built from what is stored. (Two minutes, not five: DNS is what everything else leans on.)
|
||||
*
|
||||
* What is raised: **the DNS server is not running** (AdGuard Home says so: critical); **DNS is not answering** — the test question went unanswered, or was answered with an error, in every read for
|
||||
* ten minutes (critical; one lost packet is not an outage); **protection is switched off** for an hour or more (someone paused it and forgot: warning); **a filter list has not been updated for a
|
||||
* week** (warning). "Since when" is what this app saw: the time of the first read that showed it.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { adguardState, integrations } from "../db/schema.js";
|
||||
import { createAdguardAdapter, type AdguardFilterList, type AdguardRead, type AdguardStats } from "../integrations/adguard/adapter.js";
|
||||
import type { ProbeResult } from "../integrations/adguard/dnsProbe.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 2 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 10 * 60_000;
|
||||
const STATUS_FLAG = "adguardSync";
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
const DAY = 24 * HOUR;
|
||||
/** DNS that has not answered in any read for this long is down. */
|
||||
export const DNS_FAIL_AFTER_MS = 10 * MIN;
|
||||
/** Protection off for this long is forgotten, not paused. */
|
||||
export const PROTECTION_OFF_AFTER_MS = HOUR;
|
||||
/** A filter list not updated for this long has stopped updating (AdGuard Home updates them daily unless told otherwise). */
|
||||
export const FILTER_STALE_MS = 7 * DAY;
|
||||
|
||||
export interface AdguardSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, AdguardSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, AdguardSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function adguardSyncStatuses(): Promise<Record<number, AdguardSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isAdguardFresh = (status: AdguardSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof adguardState.$inferSelect;
|
||||
|
||||
/** What is stored of one AdGuard Home, as the read gave it, plus what the app has seen between reads. */
|
||||
export interface StoredAdguard {
|
||||
version: string | null;
|
||||
running: boolean;
|
||||
protectionEnabled: boolean;
|
||||
/** When the first read that showed protection off was made; null while it is on. */
|
||||
protectionOffSince: string | null;
|
||||
dnsPort: number;
|
||||
stats: AdguardStats | null;
|
||||
filters: { enabled: boolean; lists: AdguardFilterList[] } | null;
|
||||
probe: (ProbeResult & { name: string; host: string }) | null;
|
||||
/** When the first read that showed the DNS test failing was made; null while it works. */
|
||||
probeFailSince: string | null;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
function parse<T>(stored: string | null): T | null {
|
||||
if (stored === null) return null;
|
||||
try {
|
||||
return JSON.parse(stored) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const storedOf = (row: StateRow): StoredAdguard => ({
|
||||
version: row.version,
|
||||
running: row.running,
|
||||
protectionEnabled: row.protectionEnabled,
|
||||
protectionOffSince: row.protectionOffSince,
|
||||
dnsPort: row.dnsPort,
|
||||
stats: parse<AdguardStats>(row.stats),
|
||||
filters: parse<StoredAdguard["filters"]>(row.filters),
|
||||
probe: parse<StoredAdguard["probe"]>(row.probe),
|
||||
probeFailSince: row.probeFailSince,
|
||||
updatedAt: row.updatedAt,
|
||||
});
|
||||
|
||||
/** Pure: what the app has seen between reads, carried from the last read. A thing first seen wrong starts its clock now. */
|
||||
export function carrySince(read: Pick<AdguardRead, "protectionEnabled" | "probe">, previous: Pick<StoredAdguard, "protectionOffSince" | "probeFailSince"> | null, now: Date): { protectionOffSince: string | null; probeFailSince: string | null } {
|
||||
const at = now.toISOString();
|
||||
return {
|
||||
protectionOffSince: read.protectionEnabled ? null : (previous?.protectionOffSince ?? at),
|
||||
probeFailSince: read.probe.ok ? null : (previous?.probeFailSince ?? at),
|
||||
};
|
||||
}
|
||||
|
||||
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
|
||||
export async function storeRead(integrationId: number, r: AdguardRead, now: Date = new Date()): Promise<void> {
|
||||
const [previous] = await db.select().from(adguardState).where(eq(adguardState.integrationId, integrationId)).limit(1);
|
||||
const since = carrySince(r, previous ? storedOf(previous) : null, now);
|
||||
const values = {
|
||||
version: r.version,
|
||||
running: r.running,
|
||||
protectionEnabled: r.protectionEnabled,
|
||||
protectionOffSince: since.protectionOffSince,
|
||||
dnsPort: r.dnsPort,
|
||||
stats: r.stats && JSON.stringify(r.stats),
|
||||
filters: r.filters && JSON.stringify(r.filters),
|
||||
probe: JSON.stringify(r.probe),
|
||||
probeFailSince: since.probeFailSince,
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(adguardState).values({ integrationId, ...values }).onConflictDoUpdate({ target: adguardState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one AdGuard Home and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncAdguard(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<AdguardRead>): Promise<AdguardSyncStatus> {
|
||||
let status: AdguardSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "adguard") throw new Error("not an AdGuard Home integration");
|
||||
const result = await (read ? read(loaded.config) : createAdguardAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[adguard] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[adguard] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllAdguard(now: Date = new Date()): Promise<Record<number, AdguardSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "adguard"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, AdguardSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncAdguard(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetAdguard(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface AdguardProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
|
||||
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / DAY), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
|
||||
|
||||
/** Pure: the filter lists that are switched on and have not been updated for a week (or never), while filtering is on. */
|
||||
export function staleLists(filters: StoredAdguard["filters"], nowMs: number): AdguardFilterList[] {
|
||||
if (filters === null || !filters.enabled) return [];
|
||||
return filters.lists.filter((l) => l.enabled && (l.updated === null || nowMs - Date.parse(l.updated) > FILTER_STALE_MS));
|
||||
}
|
||||
|
||||
/** Pure: what is wrong with one AdGuard Home, from what is stored for it. */
|
||||
export function adguardProblems(id: number, name: string, s: StoredAdguard, nowMs: number): AdguardProblem[] {
|
||||
const out: AdguardProblem[] = [];
|
||||
if (!s.running) {
|
||||
out.push({ key: `adguard-stopped:${id}`, severity: "critical", message: `${name}: AdGuard Home says its DNS server is not running — nothing that uses it can look up a name` });
|
||||
} else if (s.probe && !s.probe.ok && s.probeFailSince !== null && nowMs - Date.parse(s.probeFailSince) >= DNS_FAIL_AFTER_MS) {
|
||||
out.push({ key: `adguard-dns:${id}`, severity: "critical", message: `${name}: DNS has not answered a question about ${s.probe.name} for ${ago(nowMs - Date.parse(s.probeFailSince))} (asked ${s.probe.host}:${s.dnsPort}) — ${s.probe.error ?? "no answer"}` });
|
||||
}
|
||||
if (s.running && !s.protectionEnabled && s.protectionOffSince !== null && nowMs - Date.parse(s.protectionOffSince) >= PROTECTION_OFF_AFTER_MS) {
|
||||
out.push({ key: `adguard-protection:${id}`, severity: "warning", message: `${name}: AdGuard Home's protection has been switched off for ${ago(nowMs - Date.parse(s.protectionOffSince))} — nothing is being blocked` });
|
||||
}
|
||||
const stale = staleLists(s.filters, nowMs);
|
||||
if (stale.length > 0) {
|
||||
const names = stale.slice(0, 3).map((l) => l.name);
|
||||
out.push({ key: `adguard-filters:${id}`, severity: "warning", message: `${name}: ${plural(stale.length, "filter list")} ${stale.length === 1 ? "has" : "have"} not been updated for over a week — ${names.join(", ")}${stale.length > 3 ? ` and ${stale.length - 3} more` : ""}` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initAdguardSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllAdguard().catch((err) => console.error("[adguard] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 35_000);
|
||||
console.log(`[adguard] reading AdGuard Home every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -89,6 +89,8 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
sonarr: { label: "Sonarr", link: "/media" },
|
||||
radarr: { label: "Radarr", link: "/media" },
|
||||
prowlarr: { label: "Prowlarr", link: "/media" },
|
||||
adguard: { label: "AdGuard Home", link: "/adguard" },
|
||||
grafana: { label: "Grafana", link: "/grafana" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -125,12 +127,16 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "slo", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^docker-vuln(?:-high)?:(\d+):/.exec(key);
|
||||
m = /^docker-(?:vuln(?:-high)?|image-age):(\d+):/.exec(key);
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
|
||||
if (m) return { type: "speedtest", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^hetzner-(off|stuck|rescue|traffic|forecast|backup|lb|cert|fip):(\d+):/.exec(key);
|
||||
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" || m[1] === "forecast" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
|
||||
m = /^adguard-(stopped|dns|protection|filters):(\d+)/.exec(key);
|
||||
if (m) return { type: "adguard", id: Number(m[2]), category: m[1] === "stopped" || m[1] === "dns" ? "services" : "monitoring" };
|
||||
m = /^grafana-(firing|rule-error|ds):(\d+):/.exec(key);
|
||||
if (m) return { type: "grafana", id: Number(m[2]), category: m[1] === "ds" ? "integration" : "monitoring" };
|
||||
m = /^(sonarr|radarr|prowlarr)-(health|queue|disk):(\d+)/.exec(key);
|
||||
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
|
||||
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
|
||||
@@ -277,7 +283,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : origin.type === "sonarr" || origin.type === "radarr" || origin.type === "prowlarr" ? "/media" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : origin.type === "sonarr" || origin.type === "radarr" || origin.type === "prowlarr" ? "/media" : origin.type === "adguard" ? "/adguard" : origin.type === "grafana" ? "/grafana" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
@@ -287,10 +293,10 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr|adguard|grafana):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] === "images" ? " (age of the images)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
}
|
||||
}).finally(() => {
|
||||
offlineKnown(new Set());
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
import { and, eq, notInArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { dockhandVulnerabilities, integrations } from "../db/schema.js";
|
||||
import { createDockhandAdapter, type DockhandEnvironmentVulns, type DockhandVulnFinding } from "../integrations/dockhand/adapter.js";
|
||||
import { createDockhandAdapter, type DockhandEnvironmentImages, type DockhandEnvironmentVulns, type DockhandRunningImage, type DockhandVulnFinding } from "../integrations/dockhand/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
@@ -53,6 +53,27 @@ export const findingsOf = (row: Pick<StateRow, "findings">): DockhandVulnFinding
|
||||
}
|
||||
};
|
||||
|
||||
export const imagesOf = (row: Pick<StateRow, "images">): DockhandRunningImage[] | null => {
|
||||
if (row.images === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(row.images);
|
||||
return Array.isArray(v) ? (v as DockhandRunningImage[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/** Writes down which images each host's running containers use and when they were built — onto the host's row (a host whose images could not be read keeps what it had). */
|
||||
export async function storeImages(integrationId: number, environments: DockhandEnvironmentImages[], now: Date = new Date()): Promise<void> {
|
||||
for (const e of environments) {
|
||||
if (e.images === null) continue;
|
||||
await db
|
||||
.update(dockhandVulnerabilities)
|
||||
.set({ images: JSON.stringify(e.images), imagesAt: now.toISOString() })
|
||||
.where(and(eq(dockhandVulnerabilities.integrationId, integrationId), eq(dockhandVulnerabilities.environmentId, e.environmentId)));
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes down one read, replacing the last one for this integration (a host Dockhand no longer lists is dropped). */
|
||||
export async function storeRead(integrationId: number, environments: DockhandEnvironmentVulns[], now: Date = new Date()): Promise<void> {
|
||||
for (const e of environments) {
|
||||
@@ -74,13 +95,24 @@ export async function storeRead(integrationId: number, environments: DockhandEnv
|
||||
}
|
||||
|
||||
/** Reads one Dockhand's scans and stores them. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncDockerVulns(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<{ environments: DockhandEnvironmentVulns[] }>): Promise<DockerVulnSyncStatus> {
|
||||
export async function syncDockerVulns(
|
||||
integrationId: number,
|
||||
now: Date = new Date(),
|
||||
read?: (config: Record<string, unknown>) => Promise<{ environments: DockhandEnvironmentVulns[] }>,
|
||||
readImages?: (config: Record<string, unknown>) => Promise<{ environments: DockhandEnvironmentImages[] }>,
|
||||
): Promise<DockerVulnSyncStatus> {
|
||||
let status: DockerVulnSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "dockhand") throw new Error("not a Dockhand integration");
|
||||
const result = await (read ? read(loaded.config) : createDockhandAdapter(loaded.config as never).readVulnerabilities());
|
||||
await storeRead(integrationId, result.environments, now);
|
||||
// The images the containers use (for how old they are) are read beside the scans; a failure here costs only that, not the scans.
|
||||
try {
|
||||
await storeImages(integrationId, (await (readImages ? readImages(loaded.config) : createDockhandAdapter(loaded.config as never).readImages())).environments, now);
|
||||
} catch (err) {
|
||||
console.error(`[docker-vulns] couldn't read the images of integration ${integrationId}:`, err instanceof Error ? err.message : err);
|
||||
}
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
@@ -148,6 +180,45 @@ export function dockerVulnProblems(integrationId: number, rows: Pick<StateRow, "
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface DockerImageProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning";
|
||||
}
|
||||
|
||||
const DAY = 86_400_000;
|
||||
export const imageAgeDays = (created: string, nowMs: number): number => Math.floor((nowMs - Date.parse(created)) / DAY);
|
||||
|
||||
/** Pure: the running images built more than `thresholdDays` ago, oldest first (an image whose build date is not known is not judged). */
|
||||
export function oldImages(images: DockhandRunningImage[] | null, thresholdDays: number, nowMs: number): { image: string; ageDays: number; containers: string[] }[] {
|
||||
if (images === null || thresholdDays <= 0) return [];
|
||||
return images
|
||||
.filter((i): i is DockhandRunningImage & { created: string } => i.created !== null)
|
||||
.map((i) => ({ image: i.image, ageDays: imageAgeDays(i.created, nowMs), containers: i.containers }))
|
||||
.filter((i) => i.ageDays > thresholdDays)
|
||||
.sort((a, b) => b.ageDays - a.ageDays || a.image.localeCompare(b.image));
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: a warning for each Docker host that has running images built more than `thresholdDays` ago — however the scans of them came out (an image that was never scanned, or scanned clean, is just as
|
||||
* stale). 0 turns it off. Judged from the stored list; a list older than STALE_AFTER_MS or never read says nothing (the caller holds it).
|
||||
*/
|
||||
export function dockerImageAgeProblems(integrationId: number, rows: Pick<StateRow, "environmentId" | "environmentName" | "images">[], thresholdDays: number, nowMs: number): DockerImageProblem[] {
|
||||
const out: DockerImageProblem[] = [];
|
||||
if (thresholdDays <= 0) return out;
|
||||
for (const r of [...rows].sort((a, b) => a.environmentName.localeCompare(b.environmentName))) {
|
||||
const old = oldImages(imagesOf(r), thresholdDays, nowMs);
|
||||
if (old.length === 0) continue;
|
||||
const named = old.slice(0, 3).map((i) => `${i.image} (${i.ageDays} days)`);
|
||||
out.push({
|
||||
key: `docker-image-age:${integrationId}:${r.environmentId}`,
|
||||
severity: "warning",
|
||||
message: `${r.environmentName}: ${old.length} running ${old.length === 1 ? "image was" : "images were"} built more than ${thresholdDays} days ago — ${named.join(", ")}${old.length > 3 ? ` and ${old.length - 3} more` : ""}`,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initDockerVulnSync(): void {
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* Keeps what Grafana says up to date: every five minutes each Grafana integration is read — its alert rules (which are firing, which cannot be evaluated) and its data sources (with Grafana's own
|
||||
* health check of each) — and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Grafana.
|
||||
*
|
||||
* What is raised: **a rule that is firing** (critical when the rule is labelled severity critical or page, a warning otherwise — a pending one is shown, not raised), **a rule Grafana cannot
|
||||
* evaluate** (an alert that cannot alert is worse than a quiet one), and **a data source that fails its health check**. Grafana can notify about its own alerts too; with both you hear twice, so
|
||||
* this is for the ones that do not reach you any other way, or for having them in one list.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { grafanaState, integrations } from "../db/schema.js";
|
||||
import { createGrafanaAdapter, type GrafanaDatasource, type GrafanaRead, type GrafanaRule } from "../integrations/grafana/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
const STATUS_FLAG = "grafanaSync";
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
const DAY = 24 * HOUR;
|
||||
|
||||
export interface GrafanaSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, GrafanaSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, GrafanaSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function grafanaSyncStatuses(): Promise<Record<number, GrafanaSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isGrafanaFresh = (status: GrafanaSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof grafanaState.$inferSelect;
|
||||
|
||||
export interface StoredGrafana {
|
||||
version: string | null;
|
||||
/** Null when the rules could not be read. */
|
||||
rules: { total: number; items: GrafanaRule[] } | null;
|
||||
datasources: GrafanaDatasource[] | null;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
function parse<T>(stored: string | null): T | null {
|
||||
if (stored === null) return null;
|
||||
try {
|
||||
return JSON.parse(stored) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const storedOf = (row: StateRow): StoredGrafana => ({ version: row.version, rules: parse<StoredGrafana["rules"]>(row.rules), datasources: parse<GrafanaDatasource[]>(row.datasources), updatedAt: row.updatedAt });
|
||||
|
||||
/** Writes down one read, replacing the last. */
|
||||
export async function storeRead(integrationId: number, r: GrafanaRead, now: Date = new Date()): Promise<void> {
|
||||
const values = { version: r.version, rules: r.rules && JSON.stringify(r.rules), datasources: r.datasources && JSON.stringify(r.datasources), updatedAt: now.toISOString() };
|
||||
await db.insert(grafanaState).values({ integrationId, ...values }).onConflictDoUpdate({ target: grafanaState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one Grafana and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncGrafana(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<GrafanaRead>): Promise<GrafanaSyncStatus> {
|
||||
let status: GrafanaSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "grafana") throw new Error("not a Grafana integration");
|
||||
const result = await (read ? read(loaded.config) : createGrafanaAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[grafana] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[grafana] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllGrafana(now: Date = new Date()): Promise<Record<number, GrafanaSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "grafana"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, GrafanaSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncGrafana(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetGrafana(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface GrafanaProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const slug = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48);
|
||||
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
|
||||
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / DAY), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
|
||||
|
||||
export const isCriticalRule = (r: Pick<GrafanaRule, "severity">) => r.severity === "critical" || r.severity === "page";
|
||||
|
||||
/** Pure: what is wrong with one Grafana, from what is stored for it. */
|
||||
export function grafanaProblems(id: number, name: string, s: StoredGrafana, nowMs: number): GrafanaProblem[] {
|
||||
const out: GrafanaProblem[] = [];
|
||||
for (const r of [...(s.rules?.items ?? [])].sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
const ident = slug(r.uid ?? `${r.group}-${r.name}`);
|
||||
const where = [r.folder, r.group].filter(Boolean).join(" / ");
|
||||
if (r.state === "firing") {
|
||||
out.push({
|
||||
key: `grafana-firing:${id}:${ident}`,
|
||||
severity: isCriticalRule(r) ? "critical" : "warning",
|
||||
message: `${name}: the rule "${r.name}" is firing${where ? ` (${where})` : ""}${r.activeAt ? ` — for ${ago(nowMs - Date.parse(r.activeAt))}` : ""}${r.summary ? ` — ${r.summary}` : ""}`,
|
||||
});
|
||||
}
|
||||
if (r.health === "error") {
|
||||
out.push({ key: `grafana-rule-error:${id}:${ident}`, severity: "warning", message: `${name}: Grafana cannot evaluate the rule "${r.name}"${where ? ` (${where})` : ""} — ${r.lastError ?? "it reports an error"}` });
|
||||
}
|
||||
}
|
||||
for (const d of [...(s.datasources ?? [])].sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
if (d.ok === false) out.push({ key: `grafana-ds:${id}:${slug(d.uid)}`, severity: "warning", message: `${name}: the data source ${d.name} (${d.type}) fails Grafana's health check${d.message ? ` — ${d.message}` : ""}` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initGrafanaSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllGrafana().catch((err) => console.error("[grafana] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 39_000);
|
||||
console.log(`[grafana] reading Grafana every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState } from "../db/schema.js";
|
||||
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState , adguardState , grafanaState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -28,10 +28,12 @@ import { isPaperlessFresh, paperlessProblems, paperlessSyncStatuses, type Paperl
|
||||
import { isN8nFresh, n8nProblems, n8nSyncStatuses, type N8nProblem } from "./n8nSync.js";
|
||||
import { isNetBoxFresh, netboxProblems, netboxSyncStatuses, type NetBoxProblem } from "./netboxSync.js";
|
||||
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type NextcloudProblem } from "./nextcloudSync.js";
|
||||
import { dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, type DockerVulnProblem } from "./dockerVulnSync.js";
|
||||
import { dockerImageAgeProblems, dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, STALE_AFTER_MS as DOCKER_VULN_STALE_MS, type DockerImageProblem, type DockerVulnProblem } from "./dockerVulnSync.js";
|
||||
import { expectedOf as speedtestExpected, isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestProblems, speedtestSyncStatuses, type SpeedtestProblem } from "./speedtestSync.js";
|
||||
import { hetznerProblems, hetznerSyncStatuses, imagesOf as hetznerImagesOf, loadBalancersOf as hetznerLoadBalancersOf, floatingIpsOf as hetznerFloatingIpsOf, certificatesOf as hetznerCertificatesOf, isHetznerFresh, serversOf as hetznerServersOf, type HetznerProblem } from "./hetznerSync.js";
|
||||
import { ARR_KINDS, type ArrKind } from "../integrations/arr/adapter.js";
|
||||
import { adguardProblems, adguardSyncStatuses, isAdguardFresh, storedOf as adguardStoredOf, type AdguardProblem } from "./adguardSync.js";
|
||||
import { grafanaProblems, grafanaSyncStatuses, isGrafanaFresh, storedOf as grafanaStoredOf, type GrafanaProblem } from "./grafanaSync.js";
|
||||
import { arrProblems, arrSyncStatuses, isArrFresh, storedOf as arrStoredOf, type ArrProblem } from "./arrSync.js";
|
||||
import { fragnetProblems, fragnetSyncStatuses, isFragnetFresh, optionsOf as fragnetOptions, serversOf as fragnetServersOf, type FragnetProblem } from "./fragnetSync.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
@@ -124,6 +126,8 @@ export interface HealthSnapshot {
|
||||
dockerMemory?: ContainerMemory;
|
||||
/** Critical vulnerabilities in the images of each Dockhand's Docker hosts, from its last scans, only while that reading is recent (services/dockerVulnSync.ts). */
|
||||
dockerVulns?: { integrationId: number; problems: DockerVulnProblem[] }[];
|
||||
/** Docker hosts with running images built long ago, only while the list of images is recent and the setting is on (services/dockerVulnSync.ts). */
|
||||
dockerImages?: { integrationId: number; problems: DockerImageProblem[] }[];
|
||||
/** What is wrong with each Hetzner Cloud project (a server that went off, a traffic quota nearly used, rescue mode left on, a missing backup), only while that reading is recent (services/hetznerSync.ts). */
|
||||
hetzner?: { integrationId: number; problems: HetznerProblem[] }[];
|
||||
/** What is wrong with each Speedtest Tracker (the internet slower than it is sold at, speed tests that keep failing), only while that reading is recent (services/speedtestSync.ts). */
|
||||
@@ -132,6 +136,10 @@ export interface HealthSnapshot {
|
||||
fragnet?: { integrationId: number; problems: FragnetProblem[] }[];
|
||||
/** What is wrong with each Sonarr, Radarr and Prowlarr (what it says itself, downloads that are stuck, a disk nearly full), only while that reading is recent (services/arrSync.ts). */
|
||||
arr?: { integrationId: number; kind: ArrKind; problems: ArrProblem[] }[];
|
||||
/** What is wrong with each AdGuard Home (DNS not running or not answering, protection left off, filter lists not updating), only while that reading is recent (services/adguardSync.ts). */
|
||||
adguard?: { integrationId: number; problems: AdguardProblem[] }[];
|
||||
/** What is wrong with each Grafana (rules firing or failing, data sources failing their check), only while that reading is recent (services/grafanaSync.ts). */
|
||||
grafana?: { integrationId: number; problems: GrafanaProblem[] }[];
|
||||
/** Containers to leave alone. */
|
||||
dockerIgnore?: string[];
|
||||
/** The clock the container judgements are made on (the pass's, so a test can set it). */
|
||||
@@ -273,6 +281,10 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
// Speedtest Tracker: the internet slower than it is sold at, speed tests that keep failing.
|
||||
for (const s of snapshot.speedtest ?? []) for (const p of s.problems) out.push({ key: p.key, source: `speedtest:${s.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// AdGuard Home and Grafana.
|
||||
for (const a of snapshot.adguard ?? []) for (const p of a.problems) out.push({ key: p.key, source: `adguard:${a.integrationId}`, message: p.message, severity: p.severity });
|
||||
for (const g of snapshot.grafana ?? []) for (const p of g.problems) out.push({ key: p.key, source: `grafana:${g.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// Sonarr, Radarr and Prowlarr: what they say is wrong, downloads that are stuck, disks nearly full.
|
||||
for (const a of snapshot.arr ?? []) for (const p of a.problems) out.push({ key: p.key, source: `${a.kind}:${a.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
@@ -284,6 +296,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
|
||||
// Critical vulnerabilities in the images Dockhand has scanned.
|
||||
for (const v of snapshot.dockerVulns ?? []) for (const p of v.problems) out.push({ key: p.key, source: `dockhand:${v.integrationId}:vuln`, message: p.message, severity: p.severity });
|
||||
for (const v of snapshot.dockerImages ?? []) for (const p of v.problems) out.push({ key: p.key, source: `dockhand:${v.integrationId}:images`, message: p.message, severity: p.severity });
|
||||
|
||||
// A service-level target that has used most of its month's error budget, or all of it.
|
||||
for (const f of snapshot.levels ?? []) out.push({ key: levelKey(f), source: "slo", message: f.message, severity: f.severity });
|
||||
@@ -544,6 +557,42 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
console.error("[health] couldn't judge Speedtest Tracker:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// AdGuard Home: read by its own pass every two minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsG = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "adguard"), eq(integrations.enabled, true)));
|
||||
if (rowsG.length > 0) {
|
||||
const statuses = await adguardSyncStatuses();
|
||||
for (const r of rowsG) {
|
||||
const [state] = await db.select().from(adguardState).where(eq(adguardState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isAdguardFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`adguard:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.adguard ??= []).push({ integrationId: r.id, problems: adguardProblems(r.id, r.name, adguardStoredOf(state), Date.now()) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge AdGuard Home:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Grafana: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsR = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "grafana"), eq(integrations.enabled, true)));
|
||||
if (rowsR.length > 0) {
|
||||
const statuses = await grafanaSyncStatuses();
|
||||
for (const r of rowsR) {
|
||||
const [state] = await db.select().from(grafanaState).where(eq(grafanaState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isGrafanaFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`grafana:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.grafana ??= []).push({ integrationId: r.id, problems: grafanaProblems(r.id, r.name, grafanaStoredOf(state), Date.now()) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Grafana:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Sonarr, Radarr and Prowlarr: read by their own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsA = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(and(inArray(integrations.type, ARR_KINDS), eq(integrations.enabled, true)));
|
||||
@@ -768,6 +817,23 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
}
|
||||
}
|
||||
|
||||
// How old the images the containers run are, as the same sync pass stored them (only with the setting on; a list gone stale is held, not cleared).
|
||||
if (checks.imageAgeDays > 0) {
|
||||
try {
|
||||
for (const row of await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "dockhand"), eq(integrations.enabled, true)))) {
|
||||
const stored = await db.select().from(dockhandVulnerabilities).where(eq(dockhandVulnerabilities.integrationId, row.id));
|
||||
const fresh = stored.filter((r) => r.images !== null && r.imagesAt !== null && Date.now() - Date.parse(r.imagesAt) <= DOCKER_VULN_STALE_MS);
|
||||
if (stored.length === 0 || fresh.length < stored.filter((r) => r.images !== null).length || fresh.length === 0) {
|
||||
held.add(`dockhand:${row.id}:images`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.dockerImages ??= []).push({ integrationId: row.id, problems: dockerImageAgeProblems(row.id, fresh, checks.imageAgeDays, Date.now()) });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge the age of the images:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
}
|
||||
|
||||
// Service-level targets: how much of the month's error budget is used. A failure here costs only this check.
|
||||
try {
|
||||
snapshot.levels = await levelFindings();
|
||||
|
||||
@@ -6,7 +6,7 @@ export const INTEGRATION_GROUPS: { id: string; label: string; types: string[] }[
|
||||
{ id: "infrastructure", label: "Infrastructure", types: ["proxmox", "synology", "dockhand", "hetzner", "fragnet", "tailscale"] },
|
||||
{ id: "apps", label: "Apps", types: ["homeassistant", "jellyfin", "sonarr", "radarr", "prowlarr", "immich", "paperless", "nextcloud", "authentik"] },
|
||||
{ id: "backups", label: "Backups", types: ["pbs"] },
|
||||
{ id: "network", label: "Network", types: ["unifi", "npm", "phpipam", "netbox"] },
|
||||
{ id: "monitoring", label: "Monitoring", types: ["uptimekuma", "prometheus", "ups", "speedtest"] },
|
||||
{ id: "network", label: "Network", types: ["unifi", "npm", "phpipam", "netbox", "adguard"] },
|
||||
{ id: "monitoring", label: "Monitoring", types: ["uptimekuma", "prometheus", "grafana", "ups", "speedtest"] },
|
||||
{ id: "automation", label: "Automation", types: ["semaphore", "gitea", "osticket", "n8n"] },
|
||||
];
|
||||
@@ -31,7 +31,7 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^docker-vuln(?:-high)?:(\d+):/.exec(key);
|
||||
m = /^docker-(?:vuln(?:-high)?|image-age):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
@@ -45,6 +45,10 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^(?:sonarr|radarr|prowlarr)-(?:health|queue|disk):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^adguard-(?:stopped|dns|protection|filters):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^grafana-(?:firing|rule-error|ds):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^n8n-(?:failed):(\d+)/.exec(key);
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
import { eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState, adguardState, grafanaState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
@@ -15,6 +15,8 @@ import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
|
||||
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
|
||||
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf, loadBalancersOf as hetznerLoadBalancersOf, certificatesOf as hetznerCertificatesOf, trafficForecast as hetznerTrafficForecast } from "./hetznerSync.js";
|
||||
import { ARR_KINDS } from "../integrations/arr/adapter.js";
|
||||
import { adguardSyncStatuses, isAdguardFresh, storedOf as adguardStoredOf } from "./adguardSync.js";
|
||||
import { grafanaSyncStatuses, isGrafanaFresh, storedOf as grafanaStoredOf } from "./grafanaSync.js";
|
||||
import { arrSyncStatuses, distinctDisks as arrDisks, isArrFresh, stuckItems as arrStuck, storedOf as arrStoredOf } from "./arrSync.js";
|
||||
import { fragnetSyncStatuses, isFragnetFresh, newestBackup as fragnetNewestBackup, serversOf as fragnetServersOf } from "./fragnetSync.js";
|
||||
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
|
||||
@@ -103,6 +105,47 @@ async function addSpeedtestMetrics(out: Exposition, now: Date) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each AdGuard Home, for /metrics: whether its DNS server runs and protects, whether the test question was answered and how fast, and what it has done over its statistics period. Only while the reading is recent. */
|
||||
async function addAdguardMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await adguardSyncStatuses();
|
||||
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "adguard"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(adguardState)) {
|
||||
const row = rows.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isAdguardFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const state = adguardStoredOf(s);
|
||||
const labels = { name: row.name };
|
||||
out.add("homelab_adguard_running", "gauge", "1 if AdGuard Home's DNS server is running, else 0.", state.running ? 1 : 0, labels);
|
||||
out.add("homelab_adguard_protection_enabled", "gauge", "1 if AdGuard Home's protection is switched on, else 0.", state.protectionEnabled ? 1 : 0, labels);
|
||||
if (state.probe) {
|
||||
out.add("homelab_adguard_dns_up", "gauge", "1 if AdGuard Home's DNS server answered a test question, else 0.", state.probe.ok ? 1 : 0, labels);
|
||||
if (state.probe.ok && state.probe.ms !== null) out.add("homelab_adguard_dns_probe_seconds", "gauge", "How long AdGuard Home's DNS server took to answer a test question, in seconds.", state.probe.ms / 1000, labels);
|
||||
}
|
||||
if (state.stats) {
|
||||
out.add("homelab_adguard_queries", "gauge", "DNS queries AdGuard Home answered in its statistics period.", state.stats.queries, labels);
|
||||
out.add("homelab_adguard_blocked", "gauge", "Of them, the ones it blocked.", state.stats.blocked, labels);
|
||||
if (state.stats.avgMs !== null) out.add("homelab_adguard_avg_processing_seconds", "gauge", "Average time AdGuard Home takes to process a query, in seconds.", state.stats.avgMs / 1000, labels);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Grafana, for /metrics: how many rules are firing, pending and cannot be evaluated, and how many data sources fail their health check. Only while the reading is recent. */
|
||||
async function addGrafanaMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await grafanaSyncStatuses();
|
||||
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "grafana"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(grafanaState)) {
|
||||
const row = rows.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isGrafanaFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const state = grafanaStoredOf(s);
|
||||
const labels = { name: row.name };
|
||||
if (state.rules) {
|
||||
out.add("homelab_grafana_rules", "gauge", "How many alert rules Grafana has.", state.rules.total, labels);
|
||||
out.add("homelab_grafana_rules_firing", "gauge", "How many of Grafana's alert rules are firing.", state.rules.items.filter((r) => r.state === "firing").length, labels);
|
||||
out.add("homelab_grafana_rules_error", "gauge", "How many of Grafana's alert rules cannot be evaluated.", state.rules.items.filter((r) => r.health === "error").length, labels);
|
||||
}
|
||||
if (state.datasources) out.add("homelab_grafana_datasources_failing", "gauge", "How many of Grafana's data sources fail its health check.", state.datasources.filter((d) => d.ok === false).length, labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Sonarr, Radarr and Prowlarr, for /metrics: how many health findings it has, its queue and how much of it is stuck, the missing items and the free space of each disk. Only while the reading is recent. */
|
||||
async function addArrMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await arrSyncStatuses();
|
||||
@@ -498,6 +541,8 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addSpeedtestMetrics(out, now);
|
||||
await addFragnetMetrics(out, now);
|
||||
await addArrMetrics(out, now);
|
||||
await addAdguardMetrics(out, now);
|
||||
await addGrafanaMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -146,6 +146,8 @@ export interface HealthCheckSettings {
|
||||
vulnerabilityAlerts: boolean;
|
||||
/** A Docker host whose scanned images have at least this many HIGH findings is a warning too (0 = off, the default; critical ones are alerted whatever this is, while vulnerabilityAlerts is on). */
|
||||
vulnerabilityHighThreshold: number;
|
||||
/** A Docker host with running images built more than this many days ago is a warning (0 = off, the default) — however their scans came out (services/dockerVulnSync.ts). */
|
||||
imageAgeDays: number;
|
||||
/** Containers to leave alone: a name, or "host/name"; * stands for anything. */
|
||||
containerIgnore: string[];
|
||||
}
|
||||
@@ -331,7 +333,7 @@ const DEFAULTS: AppSettings = {
|
||||
selfBackup: { enabled: true, time: "03:30", keep: 14 },
|
||||
profilePictures: { gravatar: false, identityProvider: false },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, vulnerabilityAlerts: true, vulnerabilityHighThreshold: 0, containerIgnore: [] },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, vulnerabilityAlerts: true, vulnerabilityHighThreshold: 0, imageAgeDays: 0, containerIgnore: [] },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
consistency: { excludedRanges: ["172.16.0.0/12"] },
|
||||
|
||||
@@ -0,0 +1,546 @@
|
||||
/**
|
||||
* AdGuard Home (a real DNS question over UDP, its status, statistics and filter lists) and Grafana (alert rules and data-source health): reading what their APIs say, what is stored and
|
||||
* carried between reads, what is wrong, the health pass, the alerts, the pages' routes and /metrics.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import * as dgram from "node:dgram";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-adg-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, integrations, adguardState, grafanaState } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const probe = await import("../src/integrations/adguard/dnsProbe.js");
|
||||
const adg = await import("../src/integrations/adguard/adapter.js");
|
||||
const gfa = await import("../src/integrations/grafana/adapter.js");
|
||||
const ag = await import("../src/services/adguardSync.js");
|
||||
const gs = await import("../src/services/grafanaSync.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { adguardRouter } = await import("../src/routes/adguard.js");
|
||||
const { grafanaRouter } = await import("../src/routes/grafana.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
const DAY = 24 * HOUR;
|
||||
const iso = (ms: number) => new Date(ms).toISOString();
|
||||
|
||||
/** A DNS server that answers with the rcode it is told to (or not at all), once per question. */
|
||||
async function fakeDns(behaviour: () => { rcode?: number; silent?: boolean; wrongIdFirst?: boolean }) {
|
||||
const socket = dgram.createSocket("udp4");
|
||||
const questions: string[] = [];
|
||||
socket.on("message", (msg, rinfo) => {
|
||||
const b = behaviour();
|
||||
// The question name, to show what was asked.
|
||||
let i = 12;
|
||||
const labels: string[] = [];
|
||||
while (msg[i] !== 0 && i < msg.length) {
|
||||
labels.push(msg.subarray(i + 1, i + 1 + msg[i]).toString("ascii"));
|
||||
i += msg[i] + 1;
|
||||
}
|
||||
questions.push(labels.join("."));
|
||||
if (b.silent) return;
|
||||
const reply = Buffer.from(msg);
|
||||
reply.writeUInt16BE(0x8180 | (b.rcode ?? 0), 2);
|
||||
reply.writeUInt16BE(b.rcode ? 0 : 1, 6);
|
||||
if (b.wrongIdFirst) {
|
||||
const wrong = Buffer.from(reply);
|
||||
wrong.writeUInt16BE((msg.readUInt16BE(0) + 1) & 0xffff, 0);
|
||||
socket.send(wrong, rinfo.port, rinfo.address);
|
||||
}
|
||||
socket.send(reply, rinfo.port, rinfo.address);
|
||||
});
|
||||
await new Promise<void>((resolve) => socket.bind(0, "127.0.0.1", resolve));
|
||||
return { port: socket.address().port, questions, close: () => socket.close() };
|
||||
}
|
||||
|
||||
describe("the DNS question", () => {
|
||||
test("a standard recursive question for an address: one question, the name in labels", () => {
|
||||
const q = probe.buildQuery("example.com", 0x1234);
|
||||
assert.deepEqual([q.readUInt16BE(0), q.readUInt16BE(2), q.readUInt16BE(4)], [0x1234, 0x0100, 1]);
|
||||
assert.deepEqual([...q.subarray(12)], [7, ...Buffer.from("example"), 3, ...Buffer.from("com"), 0, 0, 1, 0, 1]);
|
||||
assert.throws(() => probe.buildQuery("bad name!", 1), /not a name/);
|
||||
assert.throws(() => probe.buildQuery("", 1), /not a name/);
|
||||
});
|
||||
|
||||
test("an answer is one with the same id and the response bit; how it ended is the rcode", () => {
|
||||
const q = probe.buildQuery("example.com", 7);
|
||||
const answer = Buffer.from(q);
|
||||
answer.writeUInt16BE(0x8183, 2);
|
||||
assert.deepEqual(probe.parseResponse(answer, 7), { rcode: 3, answers: 0 });
|
||||
assert.equal(probe.parseResponse(answer, 8), null, "another question's answer");
|
||||
assert.equal(probe.parseResponse(q, 7), null, "a question, not an answer");
|
||||
assert.equal(probe.parseResponse(Buffer.alloc(5), 7), null);
|
||||
});
|
||||
|
||||
describe("against a DNS server that answers", () => {
|
||||
let behaviour: { rcode?: number; silent?: boolean; wrongIdFirst?: boolean } = {};
|
||||
let dns: Awaited<ReturnType<typeof fakeDns>>;
|
||||
before(async () => {
|
||||
dns = await fakeDns(() => behaviour);
|
||||
});
|
||||
after(() => dns.close());
|
||||
|
||||
test("an address, and 'no such name', both mean the resolver works; a server failure or a refusal does not", async () => {
|
||||
behaviour = {};
|
||||
const ok = await probe.probeDns("127.0.0.1", dns.port, "example.com", 1000);
|
||||
assert.deepEqual([ok.ok, ok.error, typeof ok.ms], [true, null, "number"]);
|
||||
assert.equal(dns.questions.at(-1), "example.com");
|
||||
behaviour = { rcode: 3 };
|
||||
assert.equal((await probe.probeDns("127.0.0.1", dns.port, "nothere.example", 1000)).ok, true);
|
||||
behaviour = { rcode: 2 };
|
||||
const fail = await probe.probeDns("127.0.0.1", dns.port, "example.com", 1000);
|
||||
assert.deepEqual([fail.ok, fail.error], [false, "answered, but with server failure (SERVFAIL)"]);
|
||||
behaviour = { rcode: 5 };
|
||||
assert.match((await probe.probeDns("127.0.0.1", dns.port, "example.com", 1000)).error ?? "", /refused/);
|
||||
});
|
||||
|
||||
test("an answer to another question is not waited on as if it were ours; silence is a timeout", async () => {
|
||||
behaviour = { wrongIdFirst: true };
|
||||
assert.equal((await probe.probeDns("127.0.0.1", dns.port, "example.com", 1000)).ok, true);
|
||||
behaviour = { silent: true };
|
||||
const r = await probe.probeDns("127.0.0.1", dns.port, "example.com", 300);
|
||||
assert.deepEqual([r.ok, r.ms, r.error], [false, null, "no answer in 0 s"]);
|
||||
behaviour = {};
|
||||
});
|
||||
|
||||
test("a name that is not one is said, not asked", async () => {
|
||||
assert.match((await probe.probeDns("127.0.0.1", dns.port, "not a name", 300)).error ?? "", /not a name/);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("what AdGuard Home's API says", () => {
|
||||
test("counts come as one-name pairs, biggest first", () => {
|
||||
assert.deepEqual(adg.parseCounted([{ "a.com": 3 }, { "b.com": 9 }, { "c.com": "x" }, "junk"]), [{ name: "b.com", count: 9 }, { name: "a.com", count: 3 }]);
|
||||
assert.deepEqual(adg.parseCounted(null), []);
|
||||
assert.equal(adg.parseCounted(Array.from({ length: 30 }, (_, i) => ({ [`d${i}.com`]: i }))).length, 10);
|
||||
});
|
||||
|
||||
test("the statistics: queries, blocked and its share, the average time in milliseconds, the period; no query count is not zero", () => {
|
||||
const s = adg.parseStats({ time_units: "hours", num_dns_queries: 2000, num_blocked_filtering: 500, num_replaced_safebrowsing: 2, num_replaced_parental: 0, num_replaced_safesearch: 1, avg_processing_time: 0.0123, dns_queries: new Array(24).fill(1), top_blocked_domains: [{ "ads.example": 400 }], top_clients: [{ "192.168.1.5": 900 }], top_queried_domains: [{ "x.com": 10 }] })!;
|
||||
assert.deepEqual([s.queries, s.blocked, s.blockedPercent, s.avgMs, s.periodHours, s.safeBrowsing, s.safeSearch], [2000, 500, 25, 12.3, 24, 2, 1]);
|
||||
assert.deepEqual([s.topBlocked[0], s.topClients[0].name], [{ name: "ads.example", count: 400 }, "192.168.1.5"]);
|
||||
assert.equal(adg.parseStats({ time_units: "hours" }), null);
|
||||
assert.equal(adg.parseStats({ num_dns_queries: 0 })!.blockedPercent, null);
|
||||
});
|
||||
|
||||
test("the filter lists: switched on or off, how many rules, when updated — the year 1 is never", () => {
|
||||
const f = adg.parseFilters({ enabled: true, interval: 24, filters: [{ name: "AdGuard DNS filter", enabled: true, rules_count: 40000, last_updated: "2026-06-09T08:00:00+02:00" }, { name: "Old", enabled: false, last_updated: "0001-01-01T00:00:00Z" }, { name: "Never", enabled: true, last_updated: "0001-01-01T00:00:00Z" }, { enabled: true }] })!;
|
||||
assert.equal(f.enabled, true);
|
||||
assert.deepEqual(f.lists.map((l) => [l.name, l.enabled, l.rules, l.updated]), [["AdGuard DNS filter", true, 40000, "2026-06-09T06:00:00.000Z"], ["Old", false, null, null], ["Never", true, null, null]]);
|
||||
assert.equal(adg.parseFilters({ filters: null })!.lists.length, 0);
|
||||
assert.equal(adg.parseFilters("html"), null);
|
||||
});
|
||||
|
||||
describe("the adapter against an AdGuard Home that answers", () => {
|
||||
let http: Server;
|
||||
let url = "";
|
||||
let dns: Awaited<ReturnType<typeof fakeDns>>;
|
||||
let dnsBehaviour: { rcode?: number; silent?: boolean } = {};
|
||||
let authOk = true;
|
||||
let statsStatus = 200;
|
||||
let running = true;
|
||||
const seen: { method: string | undefined; path: string; auth: string | undefined }[] = [];
|
||||
before(async () => {
|
||||
dns = await fakeDns(() => dnsBehaviour);
|
||||
http = createHttpServer((req, res) => {
|
||||
const u = new URL(req.url ?? "", "http://x");
|
||||
seen.push({ method: req.method, path: u.pathname, auth: req.headers.authorization });
|
||||
res.setHeader("content-type", "application/json");
|
||||
if (!authOk) {
|
||||
res.statusCode = 401;
|
||||
return void res.end("{}");
|
||||
}
|
||||
if (u.pathname === "/control/status") return void res.end(JSON.stringify({ version: "v0.107.52", running, protection_enabled: true, dns_port: dns.port, http_port: 3000, dns_addresses: ["127.0.0.1"] }));
|
||||
if (u.pathname === "/control/stats") {
|
||||
if (statsStatus !== 200) {
|
||||
res.statusCode = statsStatus;
|
||||
return void res.end("{}");
|
||||
}
|
||||
return void res.end(JSON.stringify({ time_units: "hours", num_dns_queries: 100, num_blocked_filtering: 20, avg_processing_time: 0.005, dns_queries: [1, 2], top_blocked_domains: [], top_clients: [], top_queried_domains: [] }));
|
||||
}
|
||||
if (u.pathname === "/control/filtering/status") return void res.end(JSON.stringify({ enabled: true, filters: [{ name: "L", enabled: true, rules_count: 10, last_updated: "2026-06-09T08:00:00Z" }] }));
|
||||
res.statusCode = 404;
|
||||
res.end("{}");
|
||||
});
|
||||
await new Promise<void>((r) => http.listen(0, "127.0.0.1", r));
|
||||
url = `http://127.0.0.1:${(http.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => {
|
||||
http.close();
|
||||
dns.close();
|
||||
});
|
||||
|
||||
test("status, statistics, lists and the DNS question — the sign-in as Basic, and only ever GET", async () => {
|
||||
seen.length = 0;
|
||||
const r = await adg.createAdguardAdapter({ url, username: "admin", password: "secret", probeDomain: "probe.example" }).read();
|
||||
assert.deepEqual([r.version, r.running, r.protectionEnabled, r.dnsPort === dns.port, r.stats?.queries, r.filters?.lists.length], ["v0.107.52", true, true, true, 100, 1]);
|
||||
assert.deepEqual([r.probe.ok, r.probe.name, r.probe.host], [true, "probe.example", "127.0.0.1"]);
|
||||
assert.equal(dns.questions.at(-1), "probe.example");
|
||||
assert.ok(seen.every((s) => s.method === "GET" && s.auth === `Basic ${Buffer.from("admin:secret").toString("base64")}`));
|
||||
});
|
||||
|
||||
test("no sign-in sends none; a part that cannot be read is null, not empty; an unanswered DNS question is in the answer, not a failed read", async () => {
|
||||
seen.length = 0;
|
||||
statsStatus = 500;
|
||||
dnsBehaviour = { silent: true };
|
||||
const r = await adg.createAdguardAdapter({ url }).read();
|
||||
assert.deepEqual([r.stats, r.filters?.lists.length, r.probe.ok, seen.every((s) => s.auth === undefined)], [null, 1, false, true]);
|
||||
statsStatus = 200;
|
||||
dnsBehaviour = {};
|
||||
});
|
||||
|
||||
test("a refused sign-in and an address that is not AdGuard Home are said", async () => {
|
||||
authOk = false;
|
||||
assert.match((await adg.createAdguardAdapter({ url, username: "a", password: "b" }).ping()).error ?? "", /did not accept the sign-in \(HTTP 401\)/);
|
||||
authOk = true;
|
||||
assert.equal((await adg.createAdguardAdapter({ url }).ping()).ok, true);
|
||||
await assert.rejects(() => adg.createAdguardAdapter({ url: `${url}/nowhere` }).read(), /no API at this address/);
|
||||
assert.equal((await adg.createAdguardAdapter({ url: "http://127.0.0.1:1" }).ping()).ok, false);
|
||||
});
|
||||
|
||||
test("the DNS server is asked at the host of the address, or the one named", async () => {
|
||||
const r = await adg.createAdguardAdapter({ url: url.replace("127.0.0.1", "localhost"), dnsHost: "127.0.0.1" }).read();
|
||||
assert.equal(r.probe.host, "127.0.0.1");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("what AdGuard Home is carried and judged by", () => {
|
||||
const T0 = new Date("2026-06-10T10:00:00Z");
|
||||
const read = (extra: Partial<import("../src/integrations/adguard/adapter.js").AdguardRead> = {}) => ({ version: "v0.107", running: true, protectionEnabled: true, dnsPort: 53, stats: null, filters: null, probe: { ok: true, ms: 4, error: null, name: "example.com", host: "10.0.0.2" }, ...extra });
|
||||
const stored = (extra: Partial<import("../src/services/adguardSync.js").StoredAdguard> = {}) => ({ version: "v0.107", running: true, protectionEnabled: true, protectionOffSince: null, dnsPort: 53, stats: null, filters: null, probe: { ok: true, ms: 4, error: null, name: "example.com", host: "10.0.0.2" }, probeFailSince: null, updatedAt: T0.toISOString(), ...extra });
|
||||
const NOW = T0.getTime();
|
||||
const keys = (list: ReturnType<typeof ag.adguardProblems>) => list.map((p) => `${p.key}=${p.severity}`);
|
||||
|
||||
test("since when: a thing first seen wrong starts its clock now, keeps it while it stays wrong, and loses it when it is right", () => {
|
||||
const bad = read({ protectionEnabled: false, probe: { ok: false, ms: null, error: "no answer in 3 s", name: "example.com", host: "x" } });
|
||||
const first = ag.carrySince(bad, null, T0);
|
||||
assert.deepEqual(first, { protectionOffSince: T0.toISOString(), probeFailSince: T0.toISOString() });
|
||||
const later = ag.carrySince(bad, first, new Date(NOW + 5 * MIN));
|
||||
assert.deepEqual(later, first);
|
||||
assert.deepEqual(ag.carrySince(read(), later, new Date(NOW + 9 * MIN)), { protectionOffSince: null, probeFailSince: null });
|
||||
});
|
||||
|
||||
test("a DNS server that is not running is critical, and says so once (the test question is not raised beside it)", () => {
|
||||
const p = ag.adguardProblems(1, "Home DNS", stored({ running: false, probe: { ok: false, ms: null, error: "no answer in 3 s", name: "example.com", host: "x" }, probeFailSince: iso(NOW - HOUR) }), NOW);
|
||||
assert.deepEqual(keys(p), ["adguard-stopped:1=critical"]);
|
||||
assert.match(p[0].message, /Home DNS: AdGuard Home says its DNS server is not running/);
|
||||
});
|
||||
|
||||
test("a DNS question unanswered for ten minutes is critical; for less it is one lost packet", () => {
|
||||
const failing = (ago: number) => stored({ probe: { ok: false, ms: null, error: "no answer in 3 s", name: "example.com", host: "10.0.0.2" }, probeFailSince: iso(NOW - ago) });
|
||||
assert.deepEqual(ag.adguardProblems(1, "D", failing(4 * MIN), NOW), []);
|
||||
const p = ag.adguardProblems(1, "D", failing(12 * MIN), NOW);
|
||||
assert.deepEqual(keys(p), ["adguard-dns:1=critical"]);
|
||||
assert.equal(p[0].message, "D: DNS has not answered a question about example.com for 12 minutes (asked 10.0.0.2:53) — no answer in 3 s");
|
||||
assert.deepEqual(ag.adguardProblems(1, "D", stored(), NOW), []);
|
||||
});
|
||||
|
||||
test("protection off for an hour is a warning; for less it is a pause", () => {
|
||||
assert.deepEqual(ag.adguardProblems(1, "D", stored({ protectionEnabled: false, protectionOffSince: iso(NOW - 30 * MIN) }), NOW), []);
|
||||
const p = ag.adguardProblems(1, "D", stored({ protectionEnabled: false, protectionOffSince: iso(NOW - 3 * HOUR) }), NOW);
|
||||
assert.deepEqual(keys(p), ["adguard-protection:1=warning"]);
|
||||
assert.match(p[0].message, /protection has been switched off for 3 hours — nothing is being blocked/);
|
||||
});
|
||||
|
||||
test("filter lists switched on and not updated for a week (or never) are one warning; lists switched off, or filtering off, are not judged", () => {
|
||||
const list = (name: string, enabled: boolean, ageDays: number | null) => ({ name, enabled, rules: 1, updated: ageDays === null ? null : iso(NOW - ageDays * DAY) });
|
||||
const lists = [list("fresh", true, 1), list("old", true, 9), list("never", true, null), list("off", false, 30)];
|
||||
assert.deepEqual(ag.staleLists({ enabled: true, lists }, NOW).map((l) => l.name), ["old", "never"]);
|
||||
assert.deepEqual(ag.staleLists({ enabled: false, lists }, NOW), []);
|
||||
assert.deepEqual(ag.staleLists(null, NOW), []);
|
||||
const p = ag.adguardProblems(1, "D", stored({ filters: { enabled: true, lists } }), NOW);
|
||||
assert.deepEqual(keys(p), ["adguard-filters:1=warning"]);
|
||||
assert.equal(p[0].message, "D: 2 filter lists have not been updated for over a week — old, never");
|
||||
});
|
||||
});
|
||||
|
||||
describe("what AdGuard Home has stored", () => {
|
||||
let id = 0;
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "adguard", name: "DNS", baseUrl: "http://adguard", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
});
|
||||
const row = async () => (await db.select().from(adguardState).where(eq(adguardState.integrationId, id)))[0];
|
||||
after(async () => {
|
||||
await ag.forgetAdguard(id);
|
||||
await db.delete(integrations).where(eq(integrations.id, id));
|
||||
});
|
||||
|
||||
const bad = { version: "v1", running: true, protectionEnabled: false, dnsPort: 53, stats: null, filters: null, probe: { ok: false, ms: null, error: "no answer in 3 s", name: "example.com", host: "h" } };
|
||||
|
||||
test("the state is replaced at each read, carrying the clocks between them", async () => {
|
||||
const t0 = new Date();
|
||||
await ag.storeRead(id, bad, t0);
|
||||
await ag.storeRead(id, bad, new Date(t0.getTime() + 2 * MIN));
|
||||
const s = ag.storedOf(await row());
|
||||
assert.deepEqual([s.protectionEnabled, s.protectionOffSince, s.probeFailSince, s.probe?.ok], [false, t0.toISOString(), t0.toISOString(), false]);
|
||||
await ag.storeRead(id, { ...bad, protectionEnabled: true, probe: { ...bad.probe, ok: true, ms: 3, error: null } }, new Date(t0.getTime() + 4 * MIN));
|
||||
const healed = ag.storedOf(await row());
|
||||
assert.deepEqual([healed.protectionOffSince, healed.probeFailSince], [null, null]);
|
||||
});
|
||||
|
||||
test("a read that failed is said, and what was stored is left alone", async () => {
|
||||
const ok = await ag.syncAdguard(id, new Date(), async () => ({ ...bad, version: "kept" }));
|
||||
assert.equal(ok.ok, true);
|
||||
const failed = await ag.syncAdguard(id, new Date(), async () => {
|
||||
throw new Error("connection refused");
|
||||
});
|
||||
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
|
||||
assert.equal(failed.at, ok.at);
|
||||
assert.equal(ag.storedOf(await row()).version, "kept");
|
||||
assert.equal(ag.isAdguardFresh((await ag.adguardSyncStatuses())[id], Date.now() + 11 * MIN), false, "ten minutes without a read is not believed");
|
||||
});
|
||||
});
|
||||
|
||||
describe("what Grafana's API says", () => {
|
||||
const apiRule = (name: string, extra: Record<string, unknown> = {}) => ({ name, uid: `uid-${name}`, state: "inactive", health: "ok", labels: {}, annotations: {}, alerts: [], type: "alerting", ...extra });
|
||||
const raw = (rules: unknown[]) => ({ status: "success", data: { groups: [{ name: "node", file: "Infra", rules }] } });
|
||||
|
||||
test("every rule is counted; only those firing, pending or in error are kept, with where they are, how serious and since when", () => {
|
||||
const r = gfa.parseRules(
|
||||
raw([
|
||||
apiRule("quiet"),
|
||||
apiRule("loud", { state: "firing", labels: { severity: "Critical" }, annotations: { summary: "Disk is full" }, alerts: [{ activeAt: "2026-06-10T08:00:00Z" }, { activeAt: "2026-06-10T07:00:00Z" }] }),
|
||||
apiRule("soon", { state: "pending" }),
|
||||
apiRule("broken", { health: "error", lastError: "bad query" }),
|
||||
apiRule("nodata", { health: "nodata" }),
|
||||
{ state: "firing" },
|
||||
]),
|
||||
);
|
||||
assert.equal(r.total, 5);
|
||||
assert.deepEqual(r.items.map((i) => [i.name, i.state, i.health, i.severity, i.folder, i.group]), [["loud", "firing", "ok", "critical", "Infra", "node"], ["soon", "pending", "ok", null, "Infra", "node"], ["broken", "inactive", "error", null, "Infra", "node"]]);
|
||||
assert.deepEqual([r.items[0].summary, r.items[0].activeAt, r.items[2].lastError], ["Disk is full", "2026-06-10T07:00:00.000Z", "bad query"]);
|
||||
assert.deepEqual(gfa.parseRules(null), { total: 0, items: [] });
|
||||
});
|
||||
|
||||
test("data sources: uid, name, type, default; health is OK, ERROR or not tested", () => {
|
||||
assert.deepEqual(gfa.parseDatasources([{ uid: "p1", name: "Prom", type: "prometheus", isDefault: true }, { name: "no uid" }, "junk"]), [{ uid: "p1", name: "Prom", type: "prometheus", isDefault: true }]);
|
||||
assert.deepEqual(gfa.parseDatasourceHealth({ status: "OK", message: "Data source is working" }), { ok: true, message: "Data source is working" });
|
||||
assert.deepEqual(gfa.parseDatasourceHealth({ status: "ERROR", message: "connection refused" }), { ok: false, message: "connection refused" });
|
||||
assert.deepEqual(gfa.parseDatasourceHealth({ message: "not found" }), { ok: null, message: null });
|
||||
});
|
||||
|
||||
describe("the adapter against a Grafana that answers", () => {
|
||||
let http: Server;
|
||||
let url = "";
|
||||
let tokenOk = true;
|
||||
let rulesStatus = 200;
|
||||
let listStatus = 200;
|
||||
const seen: { method: string | undefined; path: string; auth: string | undefined }[] = [];
|
||||
before(async () => {
|
||||
http = createHttpServer((req, res) => {
|
||||
const u = new URL(req.url ?? "", "http://x");
|
||||
seen.push({ method: req.method, path: u.pathname, auth: req.headers.authorization });
|
||||
res.setHeader("content-type", "application/json");
|
||||
const send = (code: number, body: unknown) => {
|
||||
res.statusCode = code;
|
||||
res.end(JSON.stringify(body));
|
||||
};
|
||||
if (!tokenOk) return send(401, { message: "invalid token" });
|
||||
if (u.pathname === "/api/health") return send(200, { database: "ok", version: "11.2.0" });
|
||||
if (u.pathname === "/api/org") return send(200, { id: 1 });
|
||||
if (u.pathname === "/api/prometheus/grafana/api/v1/rules") return rulesStatus === 200 ? send(200, raw([apiRule("loud", { state: "firing" }), apiRule("quiet")])) : send(rulesStatus, {});
|
||||
if (u.pathname === "/api/datasources") return listStatus === 200 ? send(200, [{ uid: "p1", name: "Prom", type: "prometheus" }, { uid: "l1", name: "Loki", type: "loki" }, { uid: "t1", name: "Tempo", type: "tempo" }]) : send(listStatus, {});
|
||||
if (u.pathname === "/api/datasources/uid/p1/health") return send(200, { status: "OK", message: "ok" });
|
||||
if (u.pathname === "/api/datasources/uid/l1/health") return send(200, { status: "ERROR", message: "connection refused" });
|
||||
if (u.pathname === "/api/datasources/uid/t1/health") return send(404, { message: "not found" });
|
||||
send(404, {});
|
||||
});
|
||||
await new Promise<void>((r) => http.listen(0, "127.0.0.1", r));
|
||||
url = `http://127.0.0.1:${(http.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => http.close());
|
||||
|
||||
test("version, rules, data sources and each one's health — with the token as a Bearer, and only ever GET", async () => {
|
||||
seen.length = 0;
|
||||
const r = await gfa.createGrafanaAdapter({ url, token: "glsa_x" }).read();
|
||||
assert.deepEqual([r.version, r.rules?.total, r.rules?.items.map((i) => i.name)], ["11.2.0", 2, ["loud"]]);
|
||||
assert.deepEqual(r.datasources?.map((d) => [d.name, d.ok, d.message]), [["Prom", true, "ok"], ["Loki", false, "connection refused"], ["Tempo", null, null]]);
|
||||
assert.ok(seen.every((s) => s.method === "GET" && s.auth === "Bearer glsa_x"));
|
||||
});
|
||||
|
||||
test("a part that cannot be read is null, not empty; when neither can, the read fails and says why", async () => {
|
||||
rulesStatus = 403;
|
||||
const r = await gfa.createGrafanaAdapter({ url, token: "t" }).read();
|
||||
assert.deepEqual([r.rules, r.datasources?.length], [null, 3]);
|
||||
listStatus = 403;
|
||||
await assert.rejects(() => gfa.createGrafanaAdapter({ url, token: "t" }).read(), /neither the rules nor the data sources/);
|
||||
rulesStatus = 200;
|
||||
listStatus = 200;
|
||||
});
|
||||
|
||||
test("a token that is refused and an address that is not Grafana are said", async () => {
|
||||
tokenOk = false;
|
||||
const ping = await gfa.createGrafanaAdapter({ url, token: "bad" }).ping();
|
||||
assert.equal(ping.ok, false);
|
||||
assert.match(ping.error ?? "", /did not accept the token \(HTTP 401\)/);
|
||||
tokenOk = true;
|
||||
assert.equal((await gfa.createGrafanaAdapter({ url, token: "t" }).ping()).ok, true);
|
||||
await assert.rejects(() => gfa.createGrafanaAdapter({ url: `${url}/nowhere`, token: "t" }).read(), /no API at this address/);
|
||||
assert.equal((await gfa.createGrafanaAdapter({ url: "http://127.0.0.1:1", token: "t" }).ping()).ok, false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is wrong with Grafana", () => {
|
||||
const NOW = Date.parse("2026-06-10T12:00:00Z");
|
||||
const rule = (name: string, extra: Partial<import("../src/integrations/grafana/adapter.js").GrafanaRule> = {}) => ({ name, uid: `u-${name}`, group: "node", folder: "Infra", state: "inactive", health: "ok", lastError: null, severity: null, summary: null, activeAt: null, ...extra });
|
||||
const state = (rules: ReturnType<typeof rule>[] | null, datasources: import("../src/integrations/grafana/adapter.js").GrafanaDatasource[] | null = []) => ({ version: "11", rules: rules && { total: rules.length + 5, items: rules }, datasources, updatedAt: iso(NOW) });
|
||||
const keys = (list: ReturnType<typeof gs.grafanaProblems>) => list.map((p) => `${p.key}=${p.severity}`);
|
||||
|
||||
test("a firing rule is a warning — critical when labelled critical or page — and says where, since when and why; a pending one is only shown", () => {
|
||||
const p = gs.grafanaProblems(1, "Graf", state([rule("disk", { state: "firing", severity: "critical", summary: "Disk is full", activeAt: iso(NOW - 3 * HOUR) }), rule("cpu", { state: "firing", severity: "warning" }), rule("soon", { state: "pending" }), rule("pager", { state: "firing", severity: "page" })]), NOW);
|
||||
assert.deepEqual(keys(p), ["grafana-firing:1:u-cpu=warning", "grafana-firing:1:u-disk=critical", "grafana-firing:1:u-pager=critical"]);
|
||||
assert.equal(p.find((x) => x.key.endsWith("u-disk"))!.message, 'Graf: the rule "disk" is firing (Infra / node) — for 3 hours — Disk is full');
|
||||
});
|
||||
|
||||
test("a rule Grafana cannot evaluate is a warning with the reason; a rule that is firing and broken is both", () => {
|
||||
const p = gs.grafanaProblems(1, "G", state([rule("broken", { health: "error", lastError: "bad query" }), rule("both", { state: "firing", health: "error" })]), NOW);
|
||||
assert.deepEqual(keys(p).sort(), ["grafana-firing:1:u-both=warning", "grafana-rule-error:1:u-both=warning", "grafana-rule-error:1:u-broken=warning"]);
|
||||
assert.match(p.find((x) => x.key.endsWith("u-broken"))!.message, /cannot evaluate the rule "broken" \(Infra \/ node\) — bad query/);
|
||||
});
|
||||
|
||||
test("a data source that fails its check is a warning; one that works or was not tested is not", () => {
|
||||
const ds = (uid: string, ok: boolean | null) => ({ uid, name: uid.toUpperCase(), type: "loki", isDefault: false, ok, message: ok === false ? "connection refused" : null });
|
||||
const p = gs.grafanaProblems(1, "G", state([], [ds("a", true), ds("b", false), ds("c", null)]), NOW);
|
||||
assert.deepEqual(keys(p), ["grafana-ds:1:b=warning"]);
|
||||
assert.equal(p[0].message, "G: the data source B (loki) fails Grafana's health check — connection refused");
|
||||
});
|
||||
|
||||
test("lists that could not be read say nothing; a rule without a uid is keyed by its group and name", () => {
|
||||
assert.deepEqual(gs.grafanaProblems(1, "G", state(null, null), NOW), []);
|
||||
const p = gs.grafanaProblems(1, "G", state([rule("No Uid!", { uid: null, state: "firing" })]), NOW);
|
||||
assert.equal(p[0].key, "grafana-firing:1:node-no-uid");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the health pass, the Alerts page, the pages' routes and /metrics", () => {
|
||||
let agId = 0;
|
||||
let gfId = 0;
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const fresh = async () => {
|
||||
await setInternalFlag("adguardSync", JSON.stringify({ [agId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
await setInternalFlag("grafanaSync", JSON.stringify({ [gfId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
};
|
||||
const stale = async () => {
|
||||
await setInternalFlag("adguardSync", JSON.stringify({ [agId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
|
||||
await setInternalFlag("grafanaSync", JSON.stringify({ [gfId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
|
||||
};
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
agId = (await db.insert(integrations).values({ type: "adguard", name: "Home DNS", baseUrl: "http://adguard", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
gfId = (await db.insert(integrations).values({ type: "grafana", name: "Dashboards", baseUrl: "http://grafana", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
const t0 = new Date(Date.now() - 3 * HOUR);
|
||||
const down = { version: "v0.107", running: true, protectionEnabled: false, dnsPort: 53, stats: { queries: 1000, blocked: 250, blockedPercent: 25, avgMs: 8.5, safeBrowsing: 0, parental: 0, safeSearch: 0, periodHours: 24, topBlocked: [{ name: "ads.example", count: 99 }], topClients: [], topQueried: [] }, filters: { enabled: true, lists: [{ name: "Old list", enabled: true, rules: 5, updated: iso(Date.now() - 10 * DAY) }] }, probe: { ok: false, ms: null, error: "no answer in 3 s", name: "example.com", host: "10.0.0.2" } };
|
||||
await ag.storeRead(agId, down, t0);
|
||||
await ag.storeRead(agId, down, new Date());
|
||||
await gs.storeRead(gfId, { version: "11.2", rules: { total: 12, items: [{ name: "Disk full", uid: "d1", group: "node", folder: "Infra", state: "firing", health: "ok", lastError: null, severity: "critical", summary: "Disk is full", activeAt: iso(Date.now() - HOUR) }] }, datasources: [{ uid: "l1", name: "Loki", type: "loki", isDefault: false, ok: false, message: "connection refused" }, { uid: "p1", name: "Prom", type: "prometheus", isDefault: true, ok: true, message: null }] }, new Date());
|
||||
await fresh();
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/adguard", adguardRouter);
|
||||
a.use("/api/grafana", grafanaRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("the health pass judges both from what is stored, and holds one whose last read is old; maintenance on the integration silences each", async () => {
|
||||
let { snapshot, held } = await collectSnapshot();
|
||||
assert.deepEqual([snapshot.adguard?.map((x) => x.integrationId), snapshot.grafana?.map((x) => x.integrationId)], [[agId], [gfId]]);
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const mine = list.filter((x) => /^(adguard|grafana)-/.test(x.key)).map((x) => `${x.key}=${x.severity}`).sort();
|
||||
assert.deepEqual(mine, [`adguard-dns:${agId}=critical`, `adguard-filters:${agId}=warning`, `adguard-protection:${agId}=warning`, `grafana-ds:${gfId}:l1=warning`, `grafana-firing:${gfId}:d1=critical`].sort());
|
||||
assert.equal(list.find((x) => x.key === `adguard-dns:${agId}`)!.source, `adguard:${agId}`);
|
||||
for (const k of [`adguard-dns:${agId}`, `adguard-stopped:${agId}`, `adguard-protection:${agId}`, `adguard-filters:${agId}`, `grafana-firing:${gfId}:x`, `grafana-rule-error:${gfId}:x`, `grafana-ds:${gfId}:x`]) {
|
||||
assert.equal(subjectOfConditionKey(k), `integration:${/^adguard/.test(k) ? agId : gfId}`, k);
|
||||
}
|
||||
await stale();
|
||||
({ snapshot, held } = await collectSnapshot());
|
||||
assert.deepEqual([snapshot.adguard ?? [], snapshot.grafana ?? [], held.has(`adguard:${agId}`), held.has(`grafana:${gfId}`)], [[], [], true, true]);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("the Alerts page says where to look: DNS under Services, a Grafana rule under Monitoring, a failing data source under Integrations", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const find = (key: string) => report.alerts.find((x) => x.id.endsWith(key))!;
|
||||
assert.deepEqual([find(`adguard-dns:${agId}`).category, find(`adguard-dns:${agId}`).source, find(`adguard-dns:${agId}`).link, find(`adguard-protection:${agId}`).category], ["services", "AdGuard Home", "/adguard", "monitoring"]);
|
||||
assert.deepEqual([find(`grafana-firing:${gfId}:d1`).category, find(`grafana-firing:${gfId}:d1`).link, find(`grafana-ds:${gfId}:l1`).category, find(`grafana-firing:${gfId}:d1`).target?.type], ["monitoring", "/grafana", "integration", "integration"]);
|
||||
});
|
||||
|
||||
test("the pages: AdGuard Home with its DNS question, statistics, lists and what is wrong; Grafana with its rules and data sources; an old reading says nothing is wrong", async () => {
|
||||
const a = (await (await fetch(`${base}/adguard`)).json()) as { instances: { id: number; fresh: boolean; problems: unknown[]; state: { probe: { ok: boolean }; staleLists: string[]; stats: { queries: number }; protectionOffSince: string | null } }[] };
|
||||
const adg0 = a.instances.find((x) => x.id === agId)!;
|
||||
assert.deepEqual([adg0.fresh, adg0.problems.length, adg0.state.probe.ok, adg0.state.staleLists, adg0.state.stats.queries, adg0.state.protectionOffSince !== null], [true, 3, false, ["Old list"], 1000, true]);
|
||||
const g = (await (await fetch(`${base}/grafana`)).json()) as { instances: { id: number; problems: unknown[]; state: { rules: { total: number; items: { name: string }[] }; datasources: { name: string; ok: boolean | null }[] } }[] };
|
||||
const gf0 = g.instances.find((x) => x.id === gfId)!;
|
||||
assert.deepEqual([gf0.problems.length, gf0.state.rules.total, gf0.state.rules.items[0].name, gf0.state.datasources.map((d) => [d.name, d.ok])], [2, 12, "Disk full", [["Loki", false], ["Prom", true]]]);
|
||||
await stale();
|
||||
const old = (await (await fetch(`${base}/adguard`)).json()) as typeof a;
|
||||
assert.deepEqual(old.instances.find((x) => x.id === agId)!.problems, []);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("/metrics has DNS up and how fast, protection, the queries, Grafana's rules firing and failing and the data sources failing — only while the reading is recent", async () => {
|
||||
let text = await buildMetrics(new Date());
|
||||
assert.match(text, /homelab_adguard_running\{name="Home DNS"\} 1/);
|
||||
assert.match(text, /homelab_adguard_protection_enabled\{name="Home DNS"\} 0/);
|
||||
assert.match(text, /homelab_adguard_dns_up\{name="Home DNS"\} 0/);
|
||||
assert.doesNotMatch(text, /homelab_adguard_dns_probe_seconds/, "no answer, no time");
|
||||
assert.match(text, /homelab_adguard_queries\{name="Home DNS"\} 1000/);
|
||||
assert.match(text, /homelab_adguard_avg_processing_seconds\{name="Home DNS"\} 0\.0085/);
|
||||
assert.match(text, /homelab_grafana_rules\{name="Dashboards"\} 12/);
|
||||
assert.match(text, /homelab_grafana_rules_firing\{name="Dashboards"\} 1/);
|
||||
assert.match(text, /homelab_grafana_datasources_failing\{name="Dashboards"\} 1/);
|
||||
await stale();
|
||||
text = await buildMetrics(new Date());
|
||||
assert.doesNotMatch(text, /homelab_(adguard|grafana)_/);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("taking the integrations away takes what was read with them", async () => {
|
||||
await ag.forgetAdguard(agId);
|
||||
await gs.forgetGrafana(gfId);
|
||||
assert.equal((await ag.adguardSyncStatuses())[agId], undefined);
|
||||
await db.delete(integrations).where(eq(integrations.id, agId));
|
||||
await db.delete(integrations).where(eq(integrations.id, gfId));
|
||||
assert.deepEqual(await db.select().from(adguardState).where(eq(adguardState.integrationId, agId)), []);
|
||||
assert.deepEqual(await db.select().from(grafanaState).where(eq(grafanaState.integrationId, gfId)), []);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,260 @@
|
||||
/**
|
||||
* How old the images are that Docker hosts' running containers use (Dockhand's image list and container list), kept beside the vulnerability scans, and the optional warning for a host whose
|
||||
* running images were built long ago — whatever the scans of them said.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-imageage-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, integrations, dockhandVulnerabilities } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const adapter = await import("../src/integrations/dockhand/adapter.js");
|
||||
const dv = await import("../src/services/dockerVulnSync.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { integrationsRouter } = await import("../src/routes/integrations.js");
|
||||
const { getSettings, updateSettings, setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const DAY = 86_400_000;
|
||||
const NOW = Date.parse("2026-06-10T12:00:00Z");
|
||||
const secs = (daysAgo: number) => Math.floor((Date.now() - daysAgo * DAY) / 1000);
|
||||
|
||||
describe("what Dockhand says of images", () => {
|
||||
test("Created is seconds since 1970 (Docker's list), milliseconds or a date; Docker's never (0) and anything else is not a date", () => {
|
||||
assert.equal(adapter.parseImageCreated(1719830400), "2024-07-01T10:40:00.000Z");
|
||||
assert.equal(adapter.parseImageCreated(1719830400000), "2024-07-01T10:40:00.000Z");
|
||||
assert.equal(adapter.parseImageCreated("2024-07-01T10:40:00Z"), "2024-07-01T10:40:00.000Z");
|
||||
for (const bad of [0, -5, null, "", "yesterday", {}, NaN]) assert.equal(adapter.parseImageCreated(bad), null, String(bad));
|
||||
});
|
||||
|
||||
test("the image list: id, tags and when built; one with no id is skipped, and an untagged one has no tags", () => {
|
||||
const list = adapter.parseImageList([{ Id: "sha256:aaa", RepoTags: ["nginx:1.25", "nginx:latest"], Created: 1719830400 }, { Id: "sha256:bbb", RepoTags: ["<none>:<none>"], Created: 1719830400 }, { RepoTags: ["x:1"] }, "junk"]);
|
||||
assert.deepEqual(list.map((i) => [i.id, i.tags, i.created]), [["sha256:aaa", ["nginx:1.25", "nginx:latest"], "2024-07-01T10:40:00.000Z"], ["sha256:bbb", [], "2024-07-01T10:40:00.000Z"]]);
|
||||
assert.deepEqual(adapter.parseImageList(null), []);
|
||||
});
|
||||
|
||||
test("the running containers' images are matched to the list by tag — Docker Hub's prefixes and a missing tag are not a difference — or by id; stopped containers' images do not count", () => {
|
||||
const images = [
|
||||
{ id: "sha256:aaa111", tags: ["nginx:1.25"], created: "2025-01-01T00:00:00.000Z" },
|
||||
{ id: "sha256:bbb222", tags: ["redis:latest"], created: "2024-01-01T00:00:00.000Z" },
|
||||
{ id: "sha256:ccc333", tags: ["ghcr.io/org/app:2.0"], created: "2026-05-01T00:00:00.000Z" },
|
||||
{ id: "sha256:ddd444abcdef", tags: [], created: "2023-03-03T00:00:00.000Z" },
|
||||
];
|
||||
const run = adapter.matchRunningImages(
|
||||
[
|
||||
{ name: "web", image: "nginx:1.25", state: "running" },
|
||||
{ name: "web2", image: "docker.io/library/nginx:1.25", state: "running" },
|
||||
{ name: "cache", image: "redis", state: "running" },
|
||||
{ name: "app", image: "ghcr.io/org/app:2.0", state: "running" },
|
||||
{ name: "pinned", image: "sha256:ddd444abcdef", state: "running" },
|
||||
{ name: "unknown", image: "private/thing:9", state: "running" },
|
||||
{ name: "off", image: "postgres:15", state: "exited" },
|
||||
],
|
||||
images,
|
||||
);
|
||||
assert.deepEqual(run.map((r) => [r.image, r.created, r.containers]), [
|
||||
["docker.io/library/nginx:1.25", "2025-01-01T00:00:00.000Z", ["web2"]],
|
||||
["ghcr.io/org/app:2.0", "2026-05-01T00:00:00.000Z", ["app"]],
|
||||
["nginx:1.25", "2025-01-01T00:00:00.000Z", ["web"]],
|
||||
["private/thing:9", null, ["unknown"]],
|
||||
["redis", "2024-01-01T00:00:00.000Z", ["cache"]],
|
||||
["sha256:ddd444abcdef", "2023-03-03T00:00:00.000Z", ["pinned"]],
|
||||
]);
|
||||
});
|
||||
|
||||
test("two containers of one image are one entry", () => {
|
||||
const run = adapter.matchRunningImages([{ name: "a", image: "x:1", state: "running" }, { name: "b", image: "x:1", state: "running" }], [{ id: "sha256:1", tags: ["x:1"], created: null }]);
|
||||
assert.deepEqual(run.map((r) => [r.image, r.containers]), [["x:1", ["a", "b"]]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the fake Dockhand", () => {
|
||||
let dockhand: Server;
|
||||
let url = "";
|
||||
const seen: string[] = [];
|
||||
let imagesStatus: Record<string, number> = {};
|
||||
|
||||
before(async () => {
|
||||
dockhand = createHttpServer((req, res) => {
|
||||
const u = new URL(req.url ?? "", "http://x");
|
||||
seen.push(`${req.method} ${u.pathname}${u.search}`);
|
||||
res.setHeader("content-type", "application/json");
|
||||
const env = u.searchParams.get("env") ?? "";
|
||||
if (u.pathname === "/api/environments") return void res.end(JSON.stringify([{ id: 1, name: "Jackie" }, { id: 2, name: "Anna" }]));
|
||||
if (u.pathname === "/api/images") {
|
||||
if (imagesStatus[env]) {
|
||||
res.statusCode = imagesStatus[env];
|
||||
return void res.end("{}");
|
||||
}
|
||||
return void res.end(JSON.stringify(env === "1" ? [{ Id: "sha256:a1", RepoTags: ["nginx:1.19"], Created: secs(500) }, { Id: "sha256:a2", RepoTags: ["redis:7"], Created: secs(30) }, { Id: "sha256:a3", RepoTags: ["mariadb:10"], Created: secs(700) }] : [{ Id: "sha256:b1", RepoTags: ["caddy:2"], Created: secs(10) }]));
|
||||
}
|
||||
if (u.pathname === "/api/containers") {
|
||||
return void res.end(JSON.stringify(env === "1" ? [{ id: "1", name: "web", image: "nginx:1.19", state: "running" }, { id: "2", name: "cache", image: "redis:7", state: "running" }, { id: "3", name: "db", image: "mariadb:10", state: "exited" }] : [{ id: "4", name: "proxy", image: "caddy:2", state: "running" }]));
|
||||
}
|
||||
if (u.pathname === "/api/vulnerabilities/count") return void res.end(JSON.stringify({ total: 0, summary: { critical: 0, high: 0, medium: 0, low: 0, imagesScanned: 0, totalImages: 4 } }));
|
||||
res.statusCode = 404;
|
||||
res.end("{}");
|
||||
});
|
||||
await new Promise<void>((r) => dockhand.listen(0, "127.0.0.1", r));
|
||||
url = `http://127.0.0.1:${(dockhand.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => dockhand.close());
|
||||
|
||||
test("each host's images and containers are read, only with GET; a host whose list cannot be read is unknown (null), not empty", async () => {
|
||||
seen.length = 0;
|
||||
imagesStatus = { "2": 500 };
|
||||
const read = await adapter.createDockhandAdapter({ url, token: "t" } as never).readImages();
|
||||
assert.deepEqual(read.environments.map((e) => [e.environmentName, e.images?.map((i) => [i.image, i.containers])]), [["Jackie", [["nginx:1.19", ["web"]], ["redis:7", ["cache"]]]], ["Anna", undefined]]);
|
||||
assert.ok(seen.every((s) => s.startsWith("GET ")));
|
||||
imagesStatus = {};
|
||||
});
|
||||
|
||||
describe("what is stored and what is wrong", () => {
|
||||
let id = 0;
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "dockhand", name: "Dockhand", baseUrl: url, config: JSON.stringify({ url, token: "t" }) }).returning({ id: integrations.id }))[0].id;
|
||||
});
|
||||
const rows = () => db.select().from(dockhandVulnerabilities).where(eq(dockhandVulnerabilities.integrationId, id));
|
||||
|
||||
test("a sync stores each host's running images beside its scan counts; a failure to read the images does not fail the scans, and leaves what was stored", async () => {
|
||||
const status = await dv.syncDockerVulns(id, new Date());
|
||||
assert.equal(status.ok, true);
|
||||
const stored = (await rows()).sort((a, b) => a.environmentId - b.environmentId);
|
||||
assert.deepEqual(dv.imagesOf(stored[0])?.map((i) => i.image), ["nginx:1.19", "redis:7"]);
|
||||
assert.ok(stored[0].imagesAt);
|
||||
const before = stored[0].imagesAt;
|
||||
const failed = await dv.syncDockerVulns(id, new Date(Date.now() + 1000), undefined, async () => {
|
||||
throw new Error("images are down");
|
||||
});
|
||||
assert.equal(failed.ok, true, "the scans were read");
|
||||
assert.equal((await rows()).find((r) => r.environmentId === 1)!.imagesAt, before, "the images kept what they had");
|
||||
await dv.syncDockerVulns(id, new Date(), undefined, async () => ({ environments: [{ environmentId: 1, environmentName: "Jackie", images: null }] }));
|
||||
assert.equal((await rows()).find((r) => r.environmentId === 1)!.imagesAt, before, "a host whose list was null is left alone");
|
||||
});
|
||||
|
||||
test("only images built longer ago than the number are named, oldest first; an image with no build date is not judged; 0 is off", () => {
|
||||
const images = [
|
||||
{ image: "nginx:1.19", created: new Date(NOW - 500 * DAY).toISOString(), containers: ["web"] },
|
||||
{ image: "redis:7", created: new Date(NOW - 30 * DAY).toISOString(), containers: ["cache"] },
|
||||
{ image: "old:0", created: new Date(NOW - 900 * DAY).toISOString(), containers: ["x", "y"] },
|
||||
{ image: "mystery", created: null, containers: ["z"] },
|
||||
];
|
||||
assert.deepEqual(dv.oldImages(images, 180, NOW).map((i) => [i.image, i.ageDays]), [["old:0", 900], ["nginx:1.19", 500]]);
|
||||
assert.deepEqual(dv.oldImages(images, 0, NOW), []);
|
||||
assert.deepEqual(dv.oldImages(null, 180, NOW), []);
|
||||
const p = dv.dockerImageAgeProblems(7, [{ environmentId: 1, environmentName: "Jackie", images: JSON.stringify(images) }], 180, NOW);
|
||||
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["docker-image-age:7:1", "warning"]]);
|
||||
assert.equal(p[0].message, "Jackie: 2 running images were built more than 180 days ago — old:0 (900 days), nginx:1.19 (500 days)");
|
||||
assert.equal(dv.dockerImageAgeProblems(7, [{ environmentId: 1, environmentName: "Jackie", images: JSON.stringify([images[0]]) }], 180, NOW)[0].message, "Jackie: 1 running image was built more than 180 days ago — nginx:1.19 (500 days)");
|
||||
assert.deepEqual(dv.dockerImageAgeProblems(7, [{ environmentId: 1, environmentName: "Jackie", images: JSON.stringify([images[1]]) }], 180, NOW), []);
|
||||
assert.deepEqual(dv.dockerImageAgeProblems(7, [{ environmentId: 1, environmentName: "Jackie", images: null }], 180, NOW), []);
|
||||
assert.deepEqual(dv.dockerImageAgeProblems(7, [{ environmentId: 1, environmentName: "Jackie", images: JSON.stringify(images) }], 0, NOW), []);
|
||||
const many = Array.from({ length: 5 }, (_, i) => ({ image: `i${i}:1`, created: new Date(NOW - (400 + i) * DAY).toISOString(), containers: [] }));
|
||||
assert.match(dv.dockerImageAgeProblems(7, [{ environmentId: 1, environmentName: "H", images: JSON.stringify(many) }], 180, NOW)[0].message, /— i4:1 \(404 days\), i3:1 \(403 days\), i2:1 \(402 days\) and 2 more$/);
|
||||
});
|
||||
|
||||
describe("the health pass, the Alerts page and the Docker page", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const fresh = () => setInternalFlag("dockerVulnSync", JSON.stringify({ [id]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
const ageIds = async () => {
|
||||
invalidateAlertsCache();
|
||||
return (await getAlerts(true)).alerts.map((x) => x.id).filter((i) => i.includes("docker-image-age:"));
|
||||
};
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
await dv.syncDockerVulns(id, new Date());
|
||||
await fresh();
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/integrations", integrationsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/integrations/${id}/dockhand`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("off by default: nothing is raised and nothing is held (an old setting without the number is read as 0)", async () => {
|
||||
const checks = (await getSettings()).healthChecks;
|
||||
assert.equal(checks.imageAgeDays, 0);
|
||||
const { imageAgeDays: _dropped, ...older } = checks;
|
||||
await updateSettings({ healthChecks: older as never });
|
||||
assert.equal((await getSettings()).healthChecks.imageAgeDays, 0);
|
||||
const { snapshot, held } = await collectSnapshot();
|
||||
assert.deepEqual([snapshot.dockerImages, [...held].filter((h) => h.endsWith(":images"))], [undefined, []]);
|
||||
assert.deepEqual(await ageIds(), []);
|
||||
await updateSettings({ healthChecks: checks });
|
||||
});
|
||||
|
||||
test("with a number the host's old running images are one warning; maintenance holds it; a number the images are younger than raises nothing", async () => {
|
||||
const checks = (await getSettings()).healthChecks;
|
||||
await updateSettings({ healthChecks: { ...checks, imageAgeDays: 180 } });
|
||||
const { snapshot } = await collectSnapshot();
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const c = list.find((x) => x.key === `docker-image-age:${id}:1`)!;
|
||||
assert.deepEqual([c.severity, c.source], ["warning", `dockhand:${id}:images`]);
|
||||
assert.match(c.message, /Jackie: 1 running image was built more than 180 days ago — nginx:1.19 \(\d+ days\)/);
|
||||
assert.equal(list.some((x) => x.key === `docker-image-age:${id}:2`), false, "Anna's only image is young");
|
||||
assert.equal(subjectOfConditionKey(c.key), `integration:${id}`);
|
||||
assert.deepEqual(await ageIds(), [`services:docker-image-age:${id}:1`]);
|
||||
await updateSettings({ healthChecks: { ...checks, imageAgeDays: 3000 } });
|
||||
assert.deepEqual(await ageIds(), []);
|
||||
await updateSettings({ healthChecks: checks });
|
||||
});
|
||||
|
||||
test("a list that has not been read lately is held, not cleared", async () => {
|
||||
const checks = (await getSettings()).healthChecks;
|
||||
await updateSettings({ healthChecks: { ...checks, imageAgeDays: 180 } });
|
||||
await db.update(dockhandVulnerabilities).set({ imagesAt: new Date(Date.now() - 2 * 60 * 60_000).toISOString() }).where(eq(dockhandVulnerabilities.integrationId, id));
|
||||
const { snapshot, held } = await collectSnapshot();
|
||||
assert.deepEqual([snapshot.dockerImages, held.has(`dockhand:${id}:images`)], [undefined, true]);
|
||||
await updateSettings({ healthChecks: checks });
|
||||
await dv.syncDockerVulns(id, new Date());
|
||||
});
|
||||
|
||||
test("the Docker page gets each host's oldest running images and how many are over the number", async () => {
|
||||
const checks = (await getSettings()).healthChecks;
|
||||
await updateSettings({ healthChecks: { ...checks, imageAgeDays: 180 } });
|
||||
const body = (await (await fetch(`${base}/vulnerabilities`, { headers: { connection: "close" } })).json()) as { imageAgeDays: number; environments: { environmentName: string; images: { running: number; over: number | null; undated: number; oldest: { image: string; ageDays: number }[] } | null }[] };
|
||||
assert.equal(body.imageAgeDays, 180);
|
||||
const jackie = body.environments.find((e) => e.environmentName === "Jackie")!;
|
||||
assert.deepEqual([jackie.images?.running, jackie.images?.over, jackie.images?.undated, jackie.images?.oldest.map((i) => i.image)], [2, 1, 0, ["nginx:1.19", "redis:7"]]);
|
||||
assert.ok((jackie.images?.oldest[0].ageDays ?? 0) >= 499);
|
||||
await updateSettings({ healthChecks: checks });
|
||||
const off = (await (await fetch(`${base}/vulnerabilities`, { headers: { connection: "close" } })).json()) as { environments: { environmentName: string; images: { over: number | null } | null }[] };
|
||||
assert.equal(off.environments.find((e) => e.environmentName === "Jackie")!.images?.over, null, "no number, no count over it");
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* How a person arranged a server's overview page (the cards they folded and the order of the blocks): kept on their own user record, returned with /api/me, replaced by PUT /api/me/server-page.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import type { Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-pagelayout-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { meRouter, parsePageLayout } = await import("../src/routes/me.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
describe("reading what is stored", () => {
|
||||
test("a layout is read back; nothing, nonsense and a layout with a bad name are the page as drawn", () => {
|
||||
assert.deepEqual(parsePageLayout(JSON.stringify({ folded: ["notes"], order: { notes: 10, runbooks: 0 } })), { folded: ["notes"], order: { notes: 10, runbooks: 0 } });
|
||||
const none = { folded: [], order: {} };
|
||||
assert.deepEqual(parsePageLayout(null), none);
|
||||
assert.deepEqual(parsePageLayout("not json"), none);
|
||||
assert.deepEqual(parsePageLayout(JSON.stringify({ folded: ["Not A Name"], order: {} })), none);
|
||||
assert.deepEqual(parsePageLayout(JSON.stringify({ folded: [], order: { notes: -1 } })), none);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const call = (sub: string, method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json", "x-sub": sub }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "ann", name: "Ann", role: "viewer" }, { oidcSub: "bo", name: "Bo", role: "operator" }]);
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: String(req.headers["x-sub"]) } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/me", meRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/me`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const mine = async (sub: string) => ((await (await call(sub, "GET", "")).json()) as { user: { serverPage: { folded: string[]; order: Record<string, number> } } }).user.serverPage;
|
||||
|
||||
test("a person who has not arranged anything gets the page as drawn", async () => {
|
||||
assert.deepEqual(await mine("ann"), { folded: [], order: {} });
|
||||
});
|
||||
|
||||
test("anyone signed in saves their own arrangement, and gets it back with their account — not someone else's", async () => {
|
||||
const put = await call("ann", "PUT", "/server-page", { folded: ["admin-links", "admin-links", "notes"], order: { runbooks: 0, notes: 10 } });
|
||||
assert.equal(put.status, 200);
|
||||
assert.deepEqual(((await put.json()) as { serverPage: unknown }).serverPage, { folded: ["admin-links", "notes"], order: { runbooks: 0, notes: 10 } }, "a block named twice is folded once");
|
||||
assert.deepEqual(await mine("ann"), { folded: ["admin-links", "notes"], order: { runbooks: 0, notes: 10 } });
|
||||
assert.deepEqual(await mine("bo"), { folded: [], order: {} });
|
||||
});
|
||||
|
||||
test("a name that is not a block's, a position out of range and too many blocks are refused, and change nothing", async () => {
|
||||
assert.equal((await call("ann", "PUT", "/server-page", { folded: ["Admin Links"], order: {} })).status, 400);
|
||||
assert.equal((await call("ann", "PUT", "/server-page", { folded: [], order: { notes: 5000 } })).status, 400);
|
||||
assert.equal((await call("ann", "PUT", "/server-page", { folded: Array.from({ length: 61 }, (_, i) => `b${i}`), order: {} })).status, 400);
|
||||
assert.equal((await call("ann", "PUT", "/server-page", { nonsense: true })).status, 400);
|
||||
assert.deepEqual(await mine("ann"), { folded: ["admin-links", "notes"], order: { runbooks: 0, notes: 10 } });
|
||||
});
|
||||
|
||||
test("putting the page back as drawn stores nothing at all", async () => {
|
||||
assert.equal((await call("ann", "PUT", "/server-page", { folded: [], order: {} })).status, 200);
|
||||
const [row] = await db.select({ layout: users.serverPageLayout }).from(users).where(eq(users.oidcSub, "ann"));
|
||||
assert.equal(row.layout, null);
|
||||
assert.deepEqual(await mine("ann"), { folded: [], order: {} });
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user