AdGuard Home, Grafana, Docker image age, foldable server page cards
New integration kinds adguard (status, statistics, filter lists and a real DNS question over UDP; alerts for DNS down/not running, protection left off, stale lists) and grafana (alert rules firing or failing, data-source health). Optional warning for running Docker images built more than N days ago (off by default). A server's overview cards can be folded and arranged, kept per user. Migrations 0075-0077; docs, metrics and tests included. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3d9a2310bb
commit
551eb1597a
53 files changed
+27700
-33
No files matched your search
@@ -0,0 +1,205 @@
|
||||
/**
|
||||
* Keeps what AdGuard Home says up to date: every two minutes each AdGuard Home integration is read — whether its DNS server runs and protects, the statistics, its filter lists — and one real
|
||||
* DNS question is put to its DNS server; the state is stored. Pages and alerts are built from what is stored. (Two minutes, not five: DNS is what everything else leans on.)
|
||||
*
|
||||
* What is raised: **the DNS server is not running** (AdGuard Home says so: critical); **DNS is not answering** — the test question went unanswered, or was answered with an error, in every read for
|
||||
* ten minutes (critical; one lost packet is not an outage); **protection is switched off** for an hour or more (someone paused it and forgot: warning); **a filter list has not been updated for a
|
||||
* week** (warning). "Since when" is what this app saw: the time of the first read that showed it.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { adguardState, integrations } from "../db/schema.js";
|
||||
import { createAdguardAdapter, type AdguardFilterList, type AdguardRead, type AdguardStats } from "../integrations/adguard/adapter.js";
|
||||
import type { ProbeResult } from "../integrations/adguard/dnsProbe.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 2 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 10 * 60_000;
|
||||
const STATUS_FLAG = "adguardSync";
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
const DAY = 24 * HOUR;
|
||||
/** DNS that has not answered in any read for this long is down. */
|
||||
export const DNS_FAIL_AFTER_MS = 10 * MIN;
|
||||
/** Protection off for this long is forgotten, not paused. */
|
||||
export const PROTECTION_OFF_AFTER_MS = HOUR;
|
||||
/** A filter list not updated for this long has stopped updating (AdGuard Home updates them daily unless told otherwise). */
|
||||
export const FILTER_STALE_MS = 7 * DAY;
|
||||
|
||||
export interface AdguardSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, AdguardSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, AdguardSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function adguardSyncStatuses(): Promise<Record<number, AdguardSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isAdguardFresh = (status: AdguardSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof adguardState.$inferSelect;
|
||||
|
||||
/** What is stored of one AdGuard Home, as the read gave it, plus what the app has seen between reads. */
|
||||
export interface StoredAdguard {
|
||||
version: string | null;
|
||||
running: boolean;
|
||||
protectionEnabled: boolean;
|
||||
/** When the first read that showed protection off was made; null while it is on. */
|
||||
protectionOffSince: string | null;
|
||||
dnsPort: number;
|
||||
stats: AdguardStats | null;
|
||||
filters: { enabled: boolean; lists: AdguardFilterList[] } | null;
|
||||
probe: (ProbeResult & { name: string; host: string }) | null;
|
||||
/** When the first read that showed the DNS test failing was made; null while it works. */
|
||||
probeFailSince: string | null;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
function parse<T>(stored: string | null): T | null {
|
||||
if (stored === null) return null;
|
||||
try {
|
||||
return JSON.parse(stored) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const storedOf = (row: StateRow): StoredAdguard => ({
|
||||
version: row.version,
|
||||
running: row.running,
|
||||
protectionEnabled: row.protectionEnabled,
|
||||
protectionOffSince: row.protectionOffSince,
|
||||
dnsPort: row.dnsPort,
|
||||
stats: parse<AdguardStats>(row.stats),
|
||||
filters: parse<StoredAdguard["filters"]>(row.filters),
|
||||
probe: parse<StoredAdguard["probe"]>(row.probe),
|
||||
probeFailSince: row.probeFailSince,
|
||||
updatedAt: row.updatedAt,
|
||||
});
|
||||
|
||||
/** Pure: what the app has seen between reads, carried from the last read. A thing first seen wrong starts its clock now. */
|
||||
export function carrySince(read: Pick<AdguardRead, "protectionEnabled" | "probe">, previous: Pick<StoredAdguard, "protectionOffSince" | "probeFailSince"> | null, now: Date): { protectionOffSince: string | null; probeFailSince: string | null } {
|
||||
const at = now.toISOString();
|
||||
return {
|
||||
protectionOffSince: read.protectionEnabled ? null : (previous?.protectionOffSince ?? at),
|
||||
probeFailSince: read.probe.ok ? null : (previous?.probeFailSince ?? at),
|
||||
};
|
||||
}
|
||||
|
||||
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
|
||||
export async function storeRead(integrationId: number, r: AdguardRead, now: Date = new Date()): Promise<void> {
|
||||
const [previous] = await db.select().from(adguardState).where(eq(adguardState.integrationId, integrationId)).limit(1);
|
||||
const since = carrySince(r, previous ? storedOf(previous) : null, now);
|
||||
const values = {
|
||||
version: r.version,
|
||||
running: r.running,
|
||||
protectionEnabled: r.protectionEnabled,
|
||||
protectionOffSince: since.protectionOffSince,
|
||||
dnsPort: r.dnsPort,
|
||||
stats: r.stats && JSON.stringify(r.stats),
|
||||
filters: r.filters && JSON.stringify(r.filters),
|
||||
probe: JSON.stringify(r.probe),
|
||||
probeFailSince: since.probeFailSince,
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(adguardState).values({ integrationId, ...values }).onConflictDoUpdate({ target: adguardState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one AdGuard Home and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncAdguard(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<AdguardRead>): Promise<AdguardSyncStatus> {
|
||||
let status: AdguardSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "adguard") throw new Error("not an AdGuard Home integration");
|
||||
const result = await (read ? read(loaded.config) : createAdguardAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[adguard] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[adguard] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllAdguard(now: Date = new Date()): Promise<Record<number, AdguardSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "adguard"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, AdguardSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncAdguard(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetAdguard(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface AdguardProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
|
||||
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / DAY), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
|
||||
|
||||
/** Pure: the filter lists that are switched on and have not been updated for a week (or never), while filtering is on. */
|
||||
export function staleLists(filters: StoredAdguard["filters"], nowMs: number): AdguardFilterList[] {
|
||||
if (filters === null || !filters.enabled) return [];
|
||||
return filters.lists.filter((l) => l.enabled && (l.updated === null || nowMs - Date.parse(l.updated) > FILTER_STALE_MS));
|
||||
}
|
||||
|
||||
/** Pure: what is wrong with one AdGuard Home, from what is stored for it. */
|
||||
export function adguardProblems(id: number, name: string, s: StoredAdguard, nowMs: number): AdguardProblem[] {
|
||||
const out: AdguardProblem[] = [];
|
||||
if (!s.running) {
|
||||
out.push({ key: `adguard-stopped:${id}`, severity: "critical", message: `${name}: AdGuard Home says its DNS server is not running — nothing that uses it can look up a name` });
|
||||
} else if (s.probe && !s.probe.ok && s.probeFailSince !== null && nowMs - Date.parse(s.probeFailSince) >= DNS_FAIL_AFTER_MS) {
|
||||
out.push({ key: `adguard-dns:${id}`, severity: "critical", message: `${name}: DNS has not answered a question about ${s.probe.name} for ${ago(nowMs - Date.parse(s.probeFailSince))} (asked ${s.probe.host}:${s.dnsPort}) — ${s.probe.error ?? "no answer"}` });
|
||||
}
|
||||
if (s.running && !s.protectionEnabled && s.protectionOffSince !== null && nowMs - Date.parse(s.protectionOffSince) >= PROTECTION_OFF_AFTER_MS) {
|
||||
out.push({ key: `adguard-protection:${id}`, severity: "warning", message: `${name}: AdGuard Home's protection has been switched off for ${ago(nowMs - Date.parse(s.protectionOffSince))} — nothing is being blocked` });
|
||||
}
|
||||
const stale = staleLists(s.filters, nowMs);
|
||||
if (stale.length > 0) {
|
||||
const names = stale.slice(0, 3).map((l) => l.name);
|
||||
out.push({ key: `adguard-filters:${id}`, severity: "warning", message: `${name}: ${plural(stale.length, "filter list")} ${stale.length === 1 ? "has" : "have"} not been updated for over a week — ${names.join(", ")}${stale.length > 3 ? ` and ${stale.length - 3} more` : ""}` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initAdguardSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllAdguard().catch((err) => console.error("[adguard] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 35_000);
|
||||
console.log(`[adguard] reading AdGuard Home every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -89,6 +89,8 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
sonarr: { label: "Sonarr", link: "/media" },
|
||||
radarr: { label: "Radarr", link: "/media" },
|
||||
prowlarr: { label: "Prowlarr", link: "/media" },
|
||||
adguard: { label: "AdGuard Home", link: "/adguard" },
|
||||
grafana: { label: "Grafana", link: "/grafana" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -125,12 +127,16 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "slo", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^docker-vuln(?:-high)?:(\d+):/.exec(key);
|
||||
m = /^docker-(?:vuln(?:-high)?|image-age):(\d+):/.exec(key);
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
|
||||
if (m) return { type: "speedtest", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^hetzner-(off|stuck|rescue|traffic|forecast|backup|lb|cert|fip):(\d+):/.exec(key);
|
||||
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" || m[1] === "forecast" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
|
||||
m = /^adguard-(stopped|dns|protection|filters):(\d+)/.exec(key);
|
||||
if (m) return { type: "adguard", id: Number(m[2]), category: m[1] === "stopped" || m[1] === "dns" ? "services" : "monitoring" };
|
||||
m = /^grafana-(firing|rule-error|ds):(\d+):/.exec(key);
|
||||
if (m) return { type: "grafana", id: Number(m[2]), category: m[1] === "ds" ? "integration" : "monitoring" };
|
||||
m = /^(sonarr|radarr|prowlarr)-(health|queue|disk):(\d+)/.exec(key);
|
||||
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
|
||||
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
|
||||
@@ -277,7 +283,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : origin.type === "sonarr" || origin.type === "radarr" || origin.type === "prowlarr" ? "/media" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : origin.type === "sonarr" || origin.type === "radarr" || origin.type === "prowlarr" ? "/media" : origin.type === "adguard" ? "/adguard" : origin.type === "grafana" ? "/grafana" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
@@ -287,10 +293,10 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr|adguard|grafana):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] === "images" ? " (age of the images)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
}
|
||||
}).finally(() => {
|
||||
offlineKnown(new Set());
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
import { and, eq, notInArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { dockhandVulnerabilities, integrations } from "../db/schema.js";
|
||||
import { createDockhandAdapter, type DockhandEnvironmentVulns, type DockhandVulnFinding } from "../integrations/dockhand/adapter.js";
|
||||
import { createDockhandAdapter, type DockhandEnvironmentImages, type DockhandEnvironmentVulns, type DockhandRunningImage, type DockhandVulnFinding } from "../integrations/dockhand/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
@@ -53,6 +53,27 @@ export const findingsOf = (row: Pick<StateRow, "findings">): DockhandVulnFinding
|
||||
}
|
||||
};
|
||||
|
||||
export const imagesOf = (row: Pick<StateRow, "images">): DockhandRunningImage[] | null => {
|
||||
if (row.images === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(row.images);
|
||||
return Array.isArray(v) ? (v as DockhandRunningImage[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/** Writes down which images each host's running containers use and when they were built — onto the host's row (a host whose images could not be read keeps what it had). */
|
||||
export async function storeImages(integrationId: number, environments: DockhandEnvironmentImages[], now: Date = new Date()): Promise<void> {
|
||||
for (const e of environments) {
|
||||
if (e.images === null) continue;
|
||||
await db
|
||||
.update(dockhandVulnerabilities)
|
||||
.set({ images: JSON.stringify(e.images), imagesAt: now.toISOString() })
|
||||
.where(and(eq(dockhandVulnerabilities.integrationId, integrationId), eq(dockhandVulnerabilities.environmentId, e.environmentId)));
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes down one read, replacing the last one for this integration (a host Dockhand no longer lists is dropped). */
|
||||
export async function storeRead(integrationId: number, environments: DockhandEnvironmentVulns[], now: Date = new Date()): Promise<void> {
|
||||
for (const e of environments) {
|
||||
@@ -74,13 +95,24 @@ export async function storeRead(integrationId: number, environments: DockhandEnv
|
||||
}
|
||||
|
||||
/** Reads one Dockhand's scans and stores them. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncDockerVulns(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<{ environments: DockhandEnvironmentVulns[] }>): Promise<DockerVulnSyncStatus> {
|
||||
export async function syncDockerVulns(
|
||||
integrationId: number,
|
||||
now: Date = new Date(),
|
||||
read?: (config: Record<string, unknown>) => Promise<{ environments: DockhandEnvironmentVulns[] }>,
|
||||
readImages?: (config: Record<string, unknown>) => Promise<{ environments: DockhandEnvironmentImages[] }>,
|
||||
): Promise<DockerVulnSyncStatus> {
|
||||
let status: DockerVulnSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "dockhand") throw new Error("not a Dockhand integration");
|
||||
const result = await (read ? read(loaded.config) : createDockhandAdapter(loaded.config as never).readVulnerabilities());
|
||||
await storeRead(integrationId, result.environments, now);
|
||||
// The images the containers use (for how old they are) are read beside the scans; a failure here costs only that, not the scans.
|
||||
try {
|
||||
await storeImages(integrationId, (await (readImages ? readImages(loaded.config) : createDockhandAdapter(loaded.config as never).readImages())).environments, now);
|
||||
} catch (err) {
|
||||
console.error(`[docker-vulns] couldn't read the images of integration ${integrationId}:`, err instanceof Error ? err.message : err);
|
||||
}
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
@@ -148,6 +180,45 @@ export function dockerVulnProblems(integrationId: number, rows: Pick<StateRow, "
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface DockerImageProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning";
|
||||
}
|
||||
|
||||
const DAY = 86_400_000;
|
||||
export const imageAgeDays = (created: string, nowMs: number): number => Math.floor((nowMs - Date.parse(created)) / DAY);
|
||||
|
||||
/** Pure: the running images built more than `thresholdDays` ago, oldest first (an image whose build date is not known is not judged). */
|
||||
export function oldImages(images: DockhandRunningImage[] | null, thresholdDays: number, nowMs: number): { image: string; ageDays: number; containers: string[] }[] {
|
||||
if (images === null || thresholdDays <= 0) return [];
|
||||
return images
|
||||
.filter((i): i is DockhandRunningImage & { created: string } => i.created !== null)
|
||||
.map((i) => ({ image: i.image, ageDays: imageAgeDays(i.created, nowMs), containers: i.containers }))
|
||||
.filter((i) => i.ageDays > thresholdDays)
|
||||
.sort((a, b) => b.ageDays - a.ageDays || a.image.localeCompare(b.image));
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: a warning for each Docker host that has running images built more than `thresholdDays` ago — however the scans of them came out (an image that was never scanned, or scanned clean, is just as
|
||||
* stale). 0 turns it off. Judged from the stored list; a list older than STALE_AFTER_MS or never read says nothing (the caller holds it).
|
||||
*/
|
||||
export function dockerImageAgeProblems(integrationId: number, rows: Pick<StateRow, "environmentId" | "environmentName" | "images">[], thresholdDays: number, nowMs: number): DockerImageProblem[] {
|
||||
const out: DockerImageProblem[] = [];
|
||||
if (thresholdDays <= 0) return out;
|
||||
for (const r of [...rows].sort((a, b) => a.environmentName.localeCompare(b.environmentName))) {
|
||||
const old = oldImages(imagesOf(r), thresholdDays, nowMs);
|
||||
if (old.length === 0) continue;
|
||||
const named = old.slice(0, 3).map((i) => `${i.image} (${i.ageDays} days)`);
|
||||
out.push({
|
||||
key: `docker-image-age:${integrationId}:${r.environmentId}`,
|
||||
severity: "warning",
|
||||
message: `${r.environmentName}: ${old.length} running ${old.length === 1 ? "image was" : "images were"} built more than ${thresholdDays} days ago — ${named.join(", ")}${old.length > 3 ? ` and ${old.length - 3} more` : ""}`,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initDockerVulnSync(): void {
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* Keeps what Grafana says up to date: every five minutes each Grafana integration is read — its alert rules (which are firing, which cannot be evaluated) and its data sources (with Grafana's own
|
||||
* health check of each) — and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Grafana.
|
||||
*
|
||||
* What is raised: **a rule that is firing** (critical when the rule is labelled severity critical or page, a warning otherwise — a pending one is shown, not raised), **a rule Grafana cannot
|
||||
* evaluate** (an alert that cannot alert is worse than a quiet one), and **a data source that fails its health check**. Grafana can notify about its own alerts too; with both you hear twice, so
|
||||
* this is for the ones that do not reach you any other way, or for having them in one list.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { grafanaState, integrations } from "../db/schema.js";
|
||||
import { createGrafanaAdapter, type GrafanaDatasource, type GrafanaRead, type GrafanaRule } from "../integrations/grafana/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
const STATUS_FLAG = "grafanaSync";
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
const DAY = 24 * HOUR;
|
||||
|
||||
export interface GrafanaSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, GrafanaSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, GrafanaSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function grafanaSyncStatuses(): Promise<Record<number, GrafanaSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isGrafanaFresh = (status: GrafanaSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof grafanaState.$inferSelect;
|
||||
|
||||
export interface StoredGrafana {
|
||||
version: string | null;
|
||||
/** Null when the rules could not be read. */
|
||||
rules: { total: number; items: GrafanaRule[] } | null;
|
||||
datasources: GrafanaDatasource[] | null;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
function parse<T>(stored: string | null): T | null {
|
||||
if (stored === null) return null;
|
||||
try {
|
||||
return JSON.parse(stored) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const storedOf = (row: StateRow): StoredGrafana => ({ version: row.version, rules: parse<StoredGrafana["rules"]>(row.rules), datasources: parse<GrafanaDatasource[]>(row.datasources), updatedAt: row.updatedAt });
|
||||
|
||||
/** Writes down one read, replacing the last. */
|
||||
export async function storeRead(integrationId: number, r: GrafanaRead, now: Date = new Date()): Promise<void> {
|
||||
const values = { version: r.version, rules: r.rules && JSON.stringify(r.rules), datasources: r.datasources && JSON.stringify(r.datasources), updatedAt: now.toISOString() };
|
||||
await db.insert(grafanaState).values({ integrationId, ...values }).onConflictDoUpdate({ target: grafanaState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one Grafana and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncGrafana(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<GrafanaRead>): Promise<GrafanaSyncStatus> {
|
||||
let status: GrafanaSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "grafana") throw new Error("not a Grafana integration");
|
||||
const result = await (read ? read(loaded.config) : createGrafanaAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[grafana] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[grafana] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllGrafana(now: Date = new Date()): Promise<Record<number, GrafanaSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "grafana"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, GrafanaSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncGrafana(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetGrafana(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface GrafanaProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const slug = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48);
|
||||
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
|
||||
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / DAY), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
|
||||
|
||||
export const isCriticalRule = (r: Pick<GrafanaRule, "severity">) => r.severity === "critical" || r.severity === "page";
|
||||
|
||||
/** Pure: what is wrong with one Grafana, from what is stored for it. */
|
||||
export function grafanaProblems(id: number, name: string, s: StoredGrafana, nowMs: number): GrafanaProblem[] {
|
||||
const out: GrafanaProblem[] = [];
|
||||
for (const r of [...(s.rules?.items ?? [])].sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
const ident = slug(r.uid ?? `${r.group}-${r.name}`);
|
||||
const where = [r.folder, r.group].filter(Boolean).join(" / ");
|
||||
if (r.state === "firing") {
|
||||
out.push({
|
||||
key: `grafana-firing:${id}:${ident}`,
|
||||
severity: isCriticalRule(r) ? "critical" : "warning",
|
||||
message: `${name}: the rule "${r.name}" is firing${where ? ` (${where})` : ""}${r.activeAt ? ` — for ${ago(nowMs - Date.parse(r.activeAt))}` : ""}${r.summary ? ` — ${r.summary}` : ""}`,
|
||||
});
|
||||
}
|
||||
if (r.health === "error") {
|
||||
out.push({ key: `grafana-rule-error:${id}:${ident}`, severity: "warning", message: `${name}: Grafana cannot evaluate the rule "${r.name}"${where ? ` (${where})` : ""} — ${r.lastError ?? "it reports an error"}` });
|
||||
}
|
||||
}
|
||||
for (const d of [...(s.datasources ?? [])].sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
if (d.ok === false) out.push({ key: `grafana-ds:${id}:${slug(d.uid)}`, severity: "warning", message: `${name}: the data source ${d.name} (${d.type}) fails Grafana's health check${d.message ? ` — ${d.message}` : ""}` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initGrafanaSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllGrafana().catch((err) => console.error("[grafana] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 39_000);
|
||||
console.log(`[grafana] reading Grafana every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState } from "../db/schema.js";
|
||||
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState , adguardState , grafanaState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -28,10 +28,12 @@ import { isPaperlessFresh, paperlessProblems, paperlessSyncStatuses, type Paperl
|
||||
import { isN8nFresh, n8nProblems, n8nSyncStatuses, type N8nProblem } from "./n8nSync.js";
|
||||
import { isNetBoxFresh, netboxProblems, netboxSyncStatuses, type NetBoxProblem } from "./netboxSync.js";
|
||||
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type NextcloudProblem } from "./nextcloudSync.js";
|
||||
import { dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, type DockerVulnProblem } from "./dockerVulnSync.js";
|
||||
import { dockerImageAgeProblems, dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, STALE_AFTER_MS as DOCKER_VULN_STALE_MS, type DockerImageProblem, type DockerVulnProblem } from "./dockerVulnSync.js";
|
||||
import { expectedOf as speedtestExpected, isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestProblems, speedtestSyncStatuses, type SpeedtestProblem } from "./speedtestSync.js";
|
||||
import { hetznerProblems, hetznerSyncStatuses, imagesOf as hetznerImagesOf, loadBalancersOf as hetznerLoadBalancersOf, floatingIpsOf as hetznerFloatingIpsOf, certificatesOf as hetznerCertificatesOf, isHetznerFresh, serversOf as hetznerServersOf, type HetznerProblem } from "./hetznerSync.js";
|
||||
import { ARR_KINDS, type ArrKind } from "../integrations/arr/adapter.js";
|
||||
import { adguardProblems, adguardSyncStatuses, isAdguardFresh, storedOf as adguardStoredOf, type AdguardProblem } from "./adguardSync.js";
|
||||
import { grafanaProblems, grafanaSyncStatuses, isGrafanaFresh, storedOf as grafanaStoredOf, type GrafanaProblem } from "./grafanaSync.js";
|
||||
import { arrProblems, arrSyncStatuses, isArrFresh, storedOf as arrStoredOf, type ArrProblem } from "./arrSync.js";
|
||||
import { fragnetProblems, fragnetSyncStatuses, isFragnetFresh, optionsOf as fragnetOptions, serversOf as fragnetServersOf, type FragnetProblem } from "./fragnetSync.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
@@ -124,6 +126,8 @@ export interface HealthSnapshot {
|
||||
dockerMemory?: ContainerMemory;
|
||||
/** Critical vulnerabilities in the images of each Dockhand's Docker hosts, from its last scans, only while that reading is recent (services/dockerVulnSync.ts). */
|
||||
dockerVulns?: { integrationId: number; problems: DockerVulnProblem[] }[];
|
||||
/** Docker hosts with running images built long ago, only while the list of images is recent and the setting is on (services/dockerVulnSync.ts). */
|
||||
dockerImages?: { integrationId: number; problems: DockerImageProblem[] }[];
|
||||
/** What is wrong with each Hetzner Cloud project (a server that went off, a traffic quota nearly used, rescue mode left on, a missing backup), only while that reading is recent (services/hetznerSync.ts). */
|
||||
hetzner?: { integrationId: number; problems: HetznerProblem[] }[];
|
||||
/** What is wrong with each Speedtest Tracker (the internet slower than it is sold at, speed tests that keep failing), only while that reading is recent (services/speedtestSync.ts). */
|
||||
@@ -132,6 +136,10 @@ export interface HealthSnapshot {
|
||||
fragnet?: { integrationId: number; problems: FragnetProblem[] }[];
|
||||
/** What is wrong with each Sonarr, Radarr and Prowlarr (what it says itself, downloads that are stuck, a disk nearly full), only while that reading is recent (services/arrSync.ts). */
|
||||
arr?: { integrationId: number; kind: ArrKind; problems: ArrProblem[] }[];
|
||||
/** What is wrong with each AdGuard Home (DNS not running or not answering, protection left off, filter lists not updating), only while that reading is recent (services/adguardSync.ts). */
|
||||
adguard?: { integrationId: number; problems: AdguardProblem[] }[];
|
||||
/** What is wrong with each Grafana (rules firing or failing, data sources failing their check), only while that reading is recent (services/grafanaSync.ts). */
|
||||
grafana?: { integrationId: number; problems: GrafanaProblem[] }[];
|
||||
/** Containers to leave alone. */
|
||||
dockerIgnore?: string[];
|
||||
/** The clock the container judgements are made on (the pass's, so a test can set it). */
|
||||
@@ -273,6 +281,10 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
// Speedtest Tracker: the internet slower than it is sold at, speed tests that keep failing.
|
||||
for (const s of snapshot.speedtest ?? []) for (const p of s.problems) out.push({ key: p.key, source: `speedtest:${s.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// AdGuard Home and Grafana.
|
||||
for (const a of snapshot.adguard ?? []) for (const p of a.problems) out.push({ key: p.key, source: `adguard:${a.integrationId}`, message: p.message, severity: p.severity });
|
||||
for (const g of snapshot.grafana ?? []) for (const p of g.problems) out.push({ key: p.key, source: `grafana:${g.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// Sonarr, Radarr and Prowlarr: what they say is wrong, downloads that are stuck, disks nearly full.
|
||||
for (const a of snapshot.arr ?? []) for (const p of a.problems) out.push({ key: p.key, source: `${a.kind}:${a.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
@@ -284,6 +296,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
|
||||
// Critical vulnerabilities in the images Dockhand has scanned.
|
||||
for (const v of snapshot.dockerVulns ?? []) for (const p of v.problems) out.push({ key: p.key, source: `dockhand:${v.integrationId}:vuln`, message: p.message, severity: p.severity });
|
||||
for (const v of snapshot.dockerImages ?? []) for (const p of v.problems) out.push({ key: p.key, source: `dockhand:${v.integrationId}:images`, message: p.message, severity: p.severity });
|
||||
|
||||
// A service-level target that has used most of its month's error budget, or all of it.
|
||||
for (const f of snapshot.levels ?? []) out.push({ key: levelKey(f), source: "slo", message: f.message, severity: f.severity });
|
||||
@@ -544,6 +557,42 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
console.error("[health] couldn't judge Speedtest Tracker:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// AdGuard Home: read by its own pass every two minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsG = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "adguard"), eq(integrations.enabled, true)));
|
||||
if (rowsG.length > 0) {
|
||||
const statuses = await adguardSyncStatuses();
|
||||
for (const r of rowsG) {
|
||||
const [state] = await db.select().from(adguardState).where(eq(adguardState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isAdguardFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`adguard:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.adguard ??= []).push({ integrationId: r.id, problems: adguardProblems(r.id, r.name, adguardStoredOf(state), Date.now()) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge AdGuard Home:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Grafana: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsR = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "grafana"), eq(integrations.enabled, true)));
|
||||
if (rowsR.length > 0) {
|
||||
const statuses = await grafanaSyncStatuses();
|
||||
for (const r of rowsR) {
|
||||
const [state] = await db.select().from(grafanaState).where(eq(grafanaState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isGrafanaFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`grafana:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.grafana ??= []).push({ integrationId: r.id, problems: grafanaProblems(r.id, r.name, grafanaStoredOf(state), Date.now()) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Grafana:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Sonarr, Radarr and Prowlarr: read by their own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsA = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(and(inArray(integrations.type, ARR_KINDS), eq(integrations.enabled, true)));
|
||||
@@ -768,6 +817,23 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
}
|
||||
}
|
||||
|
||||
// How old the images the containers run are, as the same sync pass stored them (only with the setting on; a list gone stale is held, not cleared).
|
||||
if (checks.imageAgeDays > 0) {
|
||||
try {
|
||||
for (const row of await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "dockhand"), eq(integrations.enabled, true)))) {
|
||||
const stored = await db.select().from(dockhandVulnerabilities).where(eq(dockhandVulnerabilities.integrationId, row.id));
|
||||
const fresh = stored.filter((r) => r.images !== null && r.imagesAt !== null && Date.now() - Date.parse(r.imagesAt) <= DOCKER_VULN_STALE_MS);
|
||||
if (stored.length === 0 || fresh.length < stored.filter((r) => r.images !== null).length || fresh.length === 0) {
|
||||
held.add(`dockhand:${row.id}:images`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.dockerImages ??= []).push({ integrationId: row.id, problems: dockerImageAgeProblems(row.id, fresh, checks.imageAgeDays, Date.now()) });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge the age of the images:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
}
|
||||
|
||||
// Service-level targets: how much of the month's error budget is used. A failure here costs only this check.
|
||||
try {
|
||||
snapshot.levels = await levelFindings();
|
||||
|
||||
@@ -6,7 +6,7 @@ export const INTEGRATION_GROUPS: { id: string; label: string; types: string[] }[
|
||||
{ id: "infrastructure", label: "Infrastructure", types: ["proxmox", "synology", "dockhand", "hetzner", "fragnet", "tailscale"] },
|
||||
{ id: "apps", label: "Apps", types: ["homeassistant", "jellyfin", "sonarr", "radarr", "prowlarr", "immich", "paperless", "nextcloud", "authentik"] },
|
||||
{ id: "backups", label: "Backups", types: ["pbs"] },
|
||||
{ id: "network", label: "Network", types: ["unifi", "npm", "phpipam", "netbox"] },
|
||||
{ id: "monitoring", label: "Monitoring", types: ["uptimekuma", "prometheus", "ups", "speedtest"] },
|
||||
{ id: "network", label: "Network", types: ["unifi", "npm", "phpipam", "netbox", "adguard"] },
|
||||
{ id: "monitoring", label: "Monitoring", types: ["uptimekuma", "prometheus", "grafana", "ups", "speedtest"] },
|
||||
{ id: "automation", label: "Automation", types: ["semaphore", "gitea", "osticket", "n8n"] },
|
||||
];
|
||||
@@ -31,7 +31,7 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^docker-vuln(?:-high)?:(\d+):/.exec(key);
|
||||
m = /^docker-(?:vuln(?:-high)?|image-age):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
@@ -45,6 +45,10 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^(?:sonarr|radarr|prowlarr)-(?:health|queue|disk):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^adguard-(?:stopped|dns|protection|filters):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^grafana-(?:firing|rule-error|ds):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^n8n-(?:failed):(\d+)/.exec(key);
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
import { eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState, adguardState, grafanaState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
@@ -15,6 +15,8 @@ import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
|
||||
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
|
||||
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf, loadBalancersOf as hetznerLoadBalancersOf, certificatesOf as hetznerCertificatesOf, trafficForecast as hetznerTrafficForecast } from "./hetznerSync.js";
|
||||
import { ARR_KINDS } from "../integrations/arr/adapter.js";
|
||||
import { adguardSyncStatuses, isAdguardFresh, storedOf as adguardStoredOf } from "./adguardSync.js";
|
||||
import { grafanaSyncStatuses, isGrafanaFresh, storedOf as grafanaStoredOf } from "./grafanaSync.js";
|
||||
import { arrSyncStatuses, distinctDisks as arrDisks, isArrFresh, stuckItems as arrStuck, storedOf as arrStoredOf } from "./arrSync.js";
|
||||
import { fragnetSyncStatuses, isFragnetFresh, newestBackup as fragnetNewestBackup, serversOf as fragnetServersOf } from "./fragnetSync.js";
|
||||
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
|
||||
@@ -103,6 +105,47 @@ async function addSpeedtestMetrics(out: Exposition, now: Date) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each AdGuard Home, for /metrics: whether its DNS server runs and protects, whether the test question was answered and how fast, and what it has done over its statistics period. Only while the reading is recent. */
|
||||
async function addAdguardMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await adguardSyncStatuses();
|
||||
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "adguard"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(adguardState)) {
|
||||
const row = rows.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isAdguardFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const state = adguardStoredOf(s);
|
||||
const labels = { name: row.name };
|
||||
out.add("homelab_adguard_running", "gauge", "1 if AdGuard Home's DNS server is running, else 0.", state.running ? 1 : 0, labels);
|
||||
out.add("homelab_adguard_protection_enabled", "gauge", "1 if AdGuard Home's protection is switched on, else 0.", state.protectionEnabled ? 1 : 0, labels);
|
||||
if (state.probe) {
|
||||
out.add("homelab_adguard_dns_up", "gauge", "1 if AdGuard Home's DNS server answered a test question, else 0.", state.probe.ok ? 1 : 0, labels);
|
||||
if (state.probe.ok && state.probe.ms !== null) out.add("homelab_adguard_dns_probe_seconds", "gauge", "How long AdGuard Home's DNS server took to answer a test question, in seconds.", state.probe.ms / 1000, labels);
|
||||
}
|
||||
if (state.stats) {
|
||||
out.add("homelab_adguard_queries", "gauge", "DNS queries AdGuard Home answered in its statistics period.", state.stats.queries, labels);
|
||||
out.add("homelab_adguard_blocked", "gauge", "Of them, the ones it blocked.", state.stats.blocked, labels);
|
||||
if (state.stats.avgMs !== null) out.add("homelab_adguard_avg_processing_seconds", "gauge", "Average time AdGuard Home takes to process a query, in seconds.", state.stats.avgMs / 1000, labels);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Grafana, for /metrics: how many rules are firing, pending and cannot be evaluated, and how many data sources fail their health check. Only while the reading is recent. */
|
||||
async function addGrafanaMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await grafanaSyncStatuses();
|
||||
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "grafana"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(grafanaState)) {
|
||||
const row = rows.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isGrafanaFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const state = grafanaStoredOf(s);
|
||||
const labels = { name: row.name };
|
||||
if (state.rules) {
|
||||
out.add("homelab_grafana_rules", "gauge", "How many alert rules Grafana has.", state.rules.total, labels);
|
||||
out.add("homelab_grafana_rules_firing", "gauge", "How many of Grafana's alert rules are firing.", state.rules.items.filter((r) => r.state === "firing").length, labels);
|
||||
out.add("homelab_grafana_rules_error", "gauge", "How many of Grafana's alert rules cannot be evaluated.", state.rules.items.filter((r) => r.health === "error").length, labels);
|
||||
}
|
||||
if (state.datasources) out.add("homelab_grafana_datasources_failing", "gauge", "How many of Grafana's data sources fail its health check.", state.datasources.filter((d) => d.ok === false).length, labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Sonarr, Radarr and Prowlarr, for /metrics: how many health findings it has, its queue and how much of it is stuck, the missing items and the free space of each disk. Only while the reading is recent. */
|
||||
async function addArrMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await arrSyncStatuses();
|
||||
@@ -498,6 +541,8 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addSpeedtestMetrics(out, now);
|
||||
await addFragnetMetrics(out, now);
|
||||
await addArrMetrics(out, now);
|
||||
await addAdguardMetrics(out, now);
|
||||
await addGrafanaMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -146,6 +146,8 @@ export interface HealthCheckSettings {
|
||||
vulnerabilityAlerts: boolean;
|
||||
/** A Docker host whose scanned images have at least this many HIGH findings is a warning too (0 = off, the default; critical ones are alerted whatever this is, while vulnerabilityAlerts is on). */
|
||||
vulnerabilityHighThreshold: number;
|
||||
/** A Docker host with running images built more than this many days ago is a warning (0 = off, the default) — however their scans came out (services/dockerVulnSync.ts). */
|
||||
imageAgeDays: number;
|
||||
/** Containers to leave alone: a name, or "host/name"; * stands for anything. */
|
||||
containerIgnore: string[];
|
||||
}
|
||||
@@ -331,7 +333,7 @@ const DEFAULTS: AppSettings = {
|
||||
selfBackup: { enabled: true, time: "03:30", keep: 14 },
|
||||
profilePictures: { gravatar: false, identityProvider: false },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, vulnerabilityAlerts: true, vulnerabilityHighThreshold: 0, containerIgnore: [] },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10, osSupportWarnDays: 90, baselineAlerts: true, containerAlerts: true, vulnerabilityAlerts: true, vulnerabilityHighThreshold: 0, imageAgeDays: 0, containerIgnore: [] },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
consistency: { excludedRanges: ["172.16.0.0/12"] },
|
||||
|
||||
Reference in new issue
Block a user