Add qBittorrent and SABnzbd: a Download clients page with stuck, disconnected, paused and full-disk alerts

Each client is read every five minutes (read-only; qBittorrent's sign-in is the only POST): speed, queue and item states, network connection or paused queue, free space, SABnzbd's failures and warnings. Shared adapter, sync service, download_state table, /api/downloads, metrics and docs.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-11-04 23:13:03 +01:00
1 parent c7775e5dd9
commit 7618bf0f0a
32 files changed
+10715 -12

No files matched your search

+5 -1
View File
@@ -91,6 +91,8 @@ const SOURCES: Record<string, { label: string; link: string }> = {
lidarr: { label: "Lidarr", link: "/media" },
readarr: { label: "Readarr", link: "/media" },
prowlarr: { label: "Prowlarr", link: "/media" },
qbittorrent: { label: "qBittorrent", link: "/downloads" },
sabnzbd: { label: "SABnzbd", link: "/downloads" },
adguard: { label: "AdGuard Home", link: "/adguard" },
technitium: { label: "Technitium DNS", link: "/technitium" },
versions: { label: "Versions", link: "/versions" },
@@ -147,6 +149,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "grafana", id: Number(m[2]), category: m[1] === "ds" ? "integration" : "monitoring" };
m = /^(sonarr|radarr|prowlarr|lidarr|readarr)-(health|queue|disk):(\d+)/.exec(key);
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
m = /^(qbittorrent|sabnzbd)-(stuck|connection|paused|stalled|disk):(\d+)/.exec(key);
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
if (m) return { type: "fragnet", id: Number(m[2]), category: m[1] === "disk" ? "disk" : m[1] === "backup" ? "backup" : "monitoring" };
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
@@ -301,7 +305,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr|lidarr|readarr|adguard|technitium|grafana):(\d+)(?::(.+))?$/.exec(h);
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr|lidarr|readarr|qbittorrent|sabnzbd|adguard|technitium|grafana):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] === "images" ? " (age of the images)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
+288
View File
@@ -0,0 +1,288 @@
/**
* Keeps what qBittorrent and SABnzbd say up to date: every five minutes each download client is read — how fast it is going, what is in its queue and in what state, whether it can reach the
* network (qBittorrent) or is paused (SABnzbd), what failed lately (SABnzbd), the room in its download folder — and the state is stored. Pages and alerts are built from what is stored, so none of
* them talks to the clients.
*
* What is raised: **downloads that are stuck** — in an error state (qBittorrent: an error, files gone missing), or stalled with nothing moving for a day (one alert for the lot, naming the first few);
* **no connection** — qBittorrent has torrents but has reported no network at all for fifteen minutes (a VPN that dropped); **paused and forgotten** — SABnzbd's queue has been paused for six hours
* with downloads waiting; **downloading nothing** — SABnzbd has work waiting and is not paused, yet nothing has come in for an hour (a news server that does not answer or does not accept the
* login); **a disk nearly full** (under 10% free, critical under 5%; qBittorrent does not say how big the disk is, so there it is under 20 GB, critical under 5 GB). "Since when" is what this
* app saw: the time of the first read that showed it.
*/
import { and, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { downloadState, integrations } from "../db/schema.js";
import { DOWNLOAD_KINDS, DOWNLOAD_NAMES, createDownloadAdapter, emptyCounts, type Connection, type DownloadDisk, type DownloadItem, type DownloadKind, type DownloadRead, type DownloadState } from "../integrations/downloads/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 5 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 20 * 60_000;
const STATUS_FLAG = "downloadsSync";
const MIN = 60_000;
const HOUR = 60 * MIN;
/** A stalled torrent that has had no activity for this long is stuck, not just slow to find a source. */
export const STALLED_AFTER_MS = 24 * HOUR;
/** No network at all for this long is a dropped connection, not a restart. */
export const DISCONNECTED_AFTER_MS = 15 * MIN;
/** A queue paused for this long, with downloads waiting, is forgotten. */
export const PAUSED_AFTER_MS = 6 * HOUR;
/** Work waiting and nothing coming in for this long is a client that cannot get anything. */
export const IDLE_AFTER_MS = HOUR;
/** A failure is news for this long. */
export const FAILED_RECENT_MS = 24 * HOUR;
export const DISK_WARN_FREE = 0.1;
export const DISK_CRITICAL_FREE = 0.05;
/** Where the disk's size is not said: free space under this is a warning, under the second figure critical. */
export const DISK_WARN_BYTES = 20 * 1024 ** 3;
export const DISK_CRITICAL_BYTES = 5 * 1024 ** 3;
/** A disk smaller than this is not a download disk (a tmpfs, a boot partition). */
export const MIN_DISK_BYTES = 2 * 1024 ** 3;
export interface DownloadSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, DownloadSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, DownloadSyncStatus>) : {};
} catch {
return {};
}
}
export async function downloadSyncStatuses(): Promise<Record<number, DownloadSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isDownloadFresh = (status: DownloadSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof downloadState.$inferSelect;
function parse<T>(stored: string | null): T | null {
if (stored === null) return null;
try {
return JSON.parse(stored) as T;
} catch {
return null;
}
}
/** What the app has seen between reads: the time of the first read that showed each of these; null while it is not so. */
export interface DownloadSince {
pausedSince: string | null;
disconnectedSince: string | null;
idleSince: string | null;
}
/** What is stored of one download client, as the read gave it, plus what the app has seen between reads. */
export interface StoredDownload {
version: string | null;
speedDown: number | null;
speedUp: number | null;
paused: boolean;
connection: Connection | null;
counts: Record<DownloadState, number>;
total: number;
items: DownloadItem[];
failed: DownloadItem[] | null;
disks: DownloadDisk[] | null;
warnings: { count: number; recent: string[] } | null;
since: DownloadSince;
updatedAt: string;
}
const NO_SINCE: DownloadSince = { pausedSince: null, disconnectedSince: null, idleSince: null };
export const storedOf = (row: StateRow): StoredDownload => ({
version: row.version,
speedDown: row.speedDown,
speedUp: row.speedUp,
paused: row.paused,
connection: row.connection === "connected" || row.connection === "firewalled" || row.connection === "disconnected" ? row.connection : null,
counts: { ...emptyCounts(), ...(parse<Record<DownloadState, number>>(row.counts) ?? {}) },
total: row.total,
items: parse<DownloadItem[]>(row.items) ?? [],
failed: parse<DownloadItem[]>(row.failed),
disks: parse<DownloadDisk[]>(row.disks),
warnings: parse<{ count: number; recent: string[] }>(row.warnings),
since: { ...NO_SINCE, ...(parse<Partial<DownloadSince>>(row.since) ?? {}) },
updatedAt: row.updatedAt,
});
/** Pure: items that have not finished and are not sitting idle — what is still to be done. */
export const pendingCount = (counts: Record<DownloadState, number>, total: number): number => total - counts.seeding - counts.completed - counts.failed;
/** Pure: work is waiting for a connection to the news server, yet nothing is coming in. */
export const isIdleWithWork = (r: Pick<DownloadRead, "paused" | "speedDown" | "counts">): boolean => !r.paused && r.speedDown === 0 && r.counts.downloading + r.counts.queued > 0;
/** Pure: what the app has seen between reads, carried from the last read. A thing first seen wrong starts its clock now. */
export function carrySince(read: Pick<DownloadRead, "paused" | "connection" | "speedDown" | "counts" | "total">, previous: DownloadSince | null, now: Date): DownloadSince {
const at = now.toISOString();
return {
pausedSince: read.paused ? (previous?.pausedSince ?? at) : null,
disconnectedSince: read.connection === "disconnected" && read.total > 0 ? (previous?.disconnectedSince ?? at) : null,
idleSince: isIdleWithWork(read) ? (previous?.idleSince ?? at) : null,
};
}
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
export async function storeRead(integrationId: number, r: DownloadRead, now: Date = new Date()): Promise<void> {
const [previous] = await db.select().from(downloadState).where(eq(downloadState.integrationId, integrationId)).limit(1);
const since = carrySince(r, previous ? storedOf(previous).since : null, now);
const values = {
version: r.version,
speedDown: r.speedDown === null ? null : Math.round(r.speedDown),
speedUp: r.speedUp === null ? null : Math.round(r.speedUp),
paused: r.paused,
connection: r.connection,
counts: JSON.stringify(r.counts),
total: r.total,
items: JSON.stringify(r.items),
failed: r.failed && JSON.stringify(r.failed),
disks: r.disks && JSON.stringify(r.disks),
warnings: r.warnings && JSON.stringify(r.warnings),
since: JSON.stringify(since),
updatedAt: now.toISOString(),
};
await db.insert(downloadState).values({ integrationId, ...values }).onConflictDoUpdate({ target: downloadState.integrationId, set: values });
}
/** Reads one download client and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncDownload(integrationId: number, now: Date = new Date(), read?: (kind: DownloadKind, config: Record<string, unknown>) => Promise<DownloadRead>): Promise<DownloadSyncStatus> {
let status: DownloadSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
const kind = loaded?.integration.type as DownloadKind | undefined;
if (!loaded || !kind || !DOWNLOAD_KINDS.includes(kind)) throw new Error("not a qBittorrent or SABnzbd integration");
const result = await (read ? read(kind, loaded.config) : createDownloadAdapter(kind, loaded.config as never).read());
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[downloads] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[downloads] couldn't record the result:", err);
}
return status;
}
export async function syncAllDownloads(now: Date = new Date()): Promise<Record<number, DownloadSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(inArray(integrations.type, DOWNLOAD_KINDS), eq(integrations.enabled, true)));
const out: Record<number, DownloadSyncStatus> = {};
for (const r of rows) out[r.id] = await syncDownload(r.id, now);
return out;
}
export async function forgetDownload(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface DownloadProblem {
key: string;
message: string;
severity: "warning" | "critical";
}
const slug = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40);
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / (24 * HOUR)), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
export const sizeText = (bytes: number): string => {
const gb = bytes / 1024 ** 3;
return gb >= 1024 ? `${(gb / 1024).toFixed(1)} TB` : `${gb.toFixed(gb >= 100 ? 0 : 1)} GB`;
};
/** Pure: how long ago anything moved in a stalled item — its last activity, or when it was added; null when neither is known. */
function idleMs(i: DownloadItem, nowMs: number): number | null {
const at = i.activityAt ?? i.addedAt;
return at === null ? null : nowMs - Date.parse(at);
}
/** Pure: whether an item is stuck — in an error state, or stalled with nothing moving for a day. */
export function isStuck(i: DownloadItem, nowMs: number): boolean {
if (i.state === "error") return true;
if (i.state !== "stalled") return false;
const idle = idleMs(i, nowMs);
return idle !== null && idle >= STALLED_AFTER_MS;
}
/** Pure: the stuck items, the ones in error first, then the longest idle. */
export const stuckItems = (items: DownloadItem[], nowMs: number): DownloadItem[] =>
items.filter((i) => isStuck(i, nowMs)).sort((a, b) => Number(b.state === "error") - Number(a.state === "error") || (idleMs(b, nowMs) ?? 0) - (idleMs(a, nowMs) ?? 0));
/** Pure: the downloads that failed in the last day. */
export const recentFailed = (failed: DownloadItem[] | null, nowMs: number): DownloadItem[] => (failed ?? []).filter((f) => f.activityAt !== null && nowMs - Date.parse(f.activityAt) <= FAILED_RECENT_MS);
/** Pure: what is wrong with one download client, from what is stored for it. */
export function downloadProblems(kind: DownloadKind, id: number, name: string, s: StoredDownload, nowMs: number): DownloadProblem[] {
const out: DownloadProblem[] = [];
const app = DOWNLOAD_NAMES[kind];
const stuck = stuckItems(s.items, nowMs);
if (stuck.length > 0) {
const names = stuck.slice(0, 3).map((i) => i.name);
const worst = stuck[0];
const why = worst.state === "error" ? (worst.rawState === "missingFiles" ? "its files are missing" : "it is in an error state") : `nothing has moved for ${ago(idleMs(worst, nowMs) ?? 0)}`;
out.push({ key: `${kind}-stuck:${id}`, severity: "warning", message: `${name}: ${plural(stuck.length, "download")} ${stuck.length === 1 ? "is" : "are"} stuck — ${names.join("; ")}${stuck.length > 3 ? `; and ${stuck.length - 3} more` : ""} (${why})` });
}
if (s.since.disconnectedSince !== null && s.connection === "disconnected" && nowMs - Date.parse(s.since.disconnectedSince) >= DISCONNECTED_AFTER_MS) {
out.push({ key: `${kind}-connection:${id}`, severity: "warning", message: `${name}: ${app} has had no connection to the network for ${ago(nowMs - Date.parse(s.since.disconnectedSince))} — if it is bound to a VPN, is the VPN up?` });
}
const pending = pendingCount(s.counts, s.total);
if (s.paused && pending > 0 && s.since.pausedSince !== null && nowMs - Date.parse(s.since.pausedSince) >= PAUSED_AFTER_MS) {
out.push({ key: `${kind}-paused:${id}`, severity: "warning", message: `${name}: ${app} has been paused for ${ago(nowMs - Date.parse(s.since.pausedSince))} with ${plural(pending, "download")} waiting` });
}
if (!s.paused && s.speedDown === 0 && s.since.idleSince !== null && nowMs - Date.parse(s.since.idleSince) >= IDLE_AFTER_MS) {
const waiting = s.counts.downloading + s.counts.queued;
out.push({
key: `${kind}-stalled:${id}`,
severity: "warning",
message: `${name}: ${app} has downloaded nothing for ${ago(nowMs - Date.parse(s.since.idleSince))} although ${plural(waiting, "download")} ${waiting === 1 ? "is" : "are"} waiting — can it reach its news server?`,
});
}
for (const d of s.disks ?? []) {
if (d.total !== null) {
if (d.total < MIN_DISK_BYTES) continue;
const free = d.free / d.total;
if (free < DISK_WARN_FREE) {
out.push({ key: `${kind}-disk:${id}:${slug(d.label)}`, severity: free < DISK_CRITICAL_FREE ? "critical" : "warning", message: `${name}: the ${d.label.toLowerCase()} of ${app} has ${sizeText(d.free)} free of ${sizeText(d.total)} (${Math.floor(free * 100)}%)` });
}
} else if (d.free < DISK_WARN_BYTES) {
out.push({ key: `${kind}-disk:${id}:${slug(d.label)}`, severity: d.free < DISK_CRITICAL_BYTES ? "critical" : "warning", message: `${name}: the ${d.label.toLowerCase()} of ${app} has only ${sizeText(d.free)} free` });
}
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initDownloadsSync(): void {
if (timer) return;
const pass = () => void syncAllDownloads().catch((err) => console.error("[downloads] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 43_000);
console.log(`[downloads] reading qBittorrent and SABnzbd every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+26 -1
View File
@@ -1,6 +1,6 @@
import { and, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { technitiumState, authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState , adguardState , grafanaState } from "../db/schema.js";
import { downloadState, technitiumState, authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState , adguardState , grafanaState } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
@@ -39,6 +39,8 @@ import { adguardProblems, adguardSyncStatuses, isAdguardFresh, storedOf as adgua
import { isTechnitiumFresh, storedOf as technitiumStoredOf, technitiumProblems, technitiumSyncStatuses, type TechnitiumProblem } from "./technitiumSync.js";
import { grafanaProblems, grafanaSyncStatuses, isGrafanaFresh, storedOf as grafanaStoredOf, type GrafanaProblem } from "./grafanaSync.js";
import { arrProblems, arrSyncStatuses, isArrFresh, storedOf as arrStoredOf, type ArrProblem } from "./arrSync.js";
import { DOWNLOAD_KINDS, type DownloadKind } from "../integrations/downloads/adapter.js";
import { downloadProblems, downloadSyncStatuses, isDownloadFresh, storedOf as downloadStoredOf, type DownloadProblem } from "./downloadsSync.js";
import { fragnetProblems, fragnetSyncStatuses, isFragnetFresh, optionsOf as fragnetOptions, serversOf as fragnetServersOf, type FragnetProblem } from "./fragnetSync.js";
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
import { recordStorageSamples } from "./storageHistory.js";
@@ -140,6 +142,8 @@ export interface HealthSnapshot {
fragnet?: { integrationId: number; problems: FragnetProblem[] }[];
/** What is wrong with each Sonarr, Radarr and Prowlarr (what it says itself, downloads that are stuck, a disk nearly full), only while that reading is recent (services/arrSync.ts). */
arr?: { integrationId: number; kind: ArrKind; problems: ArrProblem[] }[];
/** What is wrong with each qBittorrent and SABnzbd (downloads that are stuck, no connection, a queue left paused or getting nothing, a disk nearly full), only while that reading is recent (services/downloadsSync.ts). */
downloads?: { integrationId: number; kind: DownloadKind; problems: DownloadProblem[] }[];
/** What is wrong with each AdGuard Home (DNS not running or not answering, protection left off, filter lists not updating), only while that reading is recent (services/adguardSync.ts). */
adguard?: { integrationId: number; problems: AdguardProblem[] }[];
/** What is wrong with each Technitium DNS Server (DNS not answering, blocking left off, block lists not updating, zones expired or not syncing), only while that reading is recent (services/technitiumSync.ts). */
@@ -297,6 +301,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
for (const t of snapshot.technitium ?? []) for (const p of t.problems) out.push({ key: p.key, source: `technitium:${t.integrationId}`, message: p.message, severity: p.severity });
for (const g of snapshot.grafana ?? []) for (const p of g.problems) out.push({ key: p.key, source: `grafana:${g.integrationId}`, message: p.message, severity: p.severity });
// qBittorrent and SABnzbd: downloads that are stuck, no connection, a queue left paused or getting nothing, disks nearly full.
for (const d of snapshot.downloads ?? []) for (const p of d.problems) out.push({ key: p.key, source: `${d.kind}:${d.integrationId}`, message: p.message, severity: p.severity });
// Sonarr, Radarr and Prowlarr: what they say is wrong, downloads that are stuck, disks nearly full.
for (const a of snapshot.arr ?? []) for (const p of a.problems) out.push({ key: p.key, source: `${a.kind}:${a.integrationId}`, message: p.message, severity: p.severity });
@@ -686,6 +693,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
console.error("[health] couldn't judge Sonarr, Radarr and Prowlarr:", err instanceof Error ? err.message : err);
}
// qBittorrent and SABnzbd: read by their own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsD = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(and(inArray(integrations.type, DOWNLOAD_KINDS), eq(integrations.enabled, true)));
if (rowsD.length > 0) {
const statuses = await downloadSyncStatuses();
for (const r of rowsD) {
const [state] = await db.select().from(downloadState).where(eq(downloadState.integrationId, r.id)).limit(1);
if (!state || !isDownloadFresh(statuses[r.id], Date.now())) {
held.add(`${r.type}:${r.id}`);
continue;
}
(snapshot.downloads ??= []).push({ integrationId: r.id, kind: r.type as DownloadKind, problems: downloadProblems(r.type as DownloadKind, r.id, r.name, downloadStoredOf(state), Date.now()) });
}
}
} catch (err) {
console.error("[health] couldn't judge qBittorrent and SABnzbd:", err instanceof Error ? err.message : err);
}
// Fragnet game servers: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsF = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "fragnet"), eq(integrations.enabled, true)));
+1 -1
View File
@@ -4,7 +4,7 @@
*/
export const INTEGRATION_GROUPS: { id: string; label: string; types: string[] }[] = [
{ id: "infrastructure", label: "Infrastructure", types: ["proxmox", "synology", "dockhand", "hetzner", "fragnet", "tailscale"] },
{ id: "apps", label: "Apps", types: ["homeassistant", "jellyfin", "sonarr", "radarr", "lidarr", "readarr", "prowlarr", "immich", "paperless", "nextcloud", "authentik"] },
{ id: "apps", label: "Apps", types: ["homeassistant", "jellyfin", "sonarr", "radarr", "lidarr", "readarr", "prowlarr", "qbittorrent", "sabnzbd", "immich", "paperless", "nextcloud", "authentik"] },
{ id: "backups", label: "Backups", types: ["pbs"] },
{ id: "network", label: "Network", types: ["unifi", "npm", "phpipam", "netbox", "adguard", "technitium"] },
{ id: "monitoring", label: "Monitoring", types: ["uptimekuma", "prometheus", "grafana", "ups", "speedtest"] },
+2
View File
@@ -45,6 +45,8 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `integration:${m[1]}`;
m = /^(?:sonarr|radarr|prowlarr|lidarr|readarr)-(?:health|queue|disk):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^(?:qbittorrent|sabnzbd)-(?:stuck|connection|paused|stalled|disk):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^version-behind:(\d+)$/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^adguard-(?:stopped|dns|protection|filters):(\d+)/.exec(key);
+24 -1
View File
@@ -5,7 +5,7 @@
*/
import { eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { technitiumState, assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState, adguardState, grafanaState } from "../db/schema.js";
import { downloadState, technitiumState, assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState, adguardState, grafanaState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { backupState, listAll as listDuplicati } from "./duplicati.js";
@@ -15,6 +15,8 @@ import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf, loadBalancersOf as hetznerLoadBalancersOf, certificatesOf as hetznerCertificatesOf, trafficForecast as hetznerTrafficForecast } from "./hetznerSync.js";
import { ARR_KINDS } from "../integrations/arr/adapter.js";
import { DOWNLOAD_KINDS, DOWNLOAD_STATES } from "../integrations/downloads/adapter.js";
import { downloadSyncStatuses, isDownloadFresh, recentFailed as downloadsFailed, storedOf as downloadStoredOf, stuckItems as downloadStuck } from "./downloadsSync.js";
import { versionRows as appVersionRows } from "./appVersions.js";
import { adguardSyncStatuses, isAdguardFresh, storedOf as adguardStoredOf } from "./adguardSync.js";
import { blockListsOverdueMs, isTechnitiumFresh, storedOf as technitiumStoredOf, technitiumSyncStatuses } from "./technitiumSync.js";
@@ -210,6 +212,26 @@ async function addArrMetrics(out: Exposition, now: Date) {
}
}
/** Each qBittorrent and SABnzbd, for /metrics: how fast it is going, the items in each state and how many are stuck, whether it is connected or paused, what failed in the last day and the free space of its disk. Only while the reading is recent. */
async function addDownloadMetrics(out: Exposition, now: Date) {
const statuses = await downloadSyncStatuses();
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations).where(inArray(integrations.type, DOWNLOAD_KINDS))).map((r) => [r.id, r]));
for (const s of await db.select().from(downloadState)) {
const row = rows.get(s.integrationId);
if (!row || !row.enabled || !isDownloadFresh(statuses[s.integrationId], now.getTime())) continue;
const state = downloadStoredOf(s);
const labels = { client: row.type, name: row.name };
if (state.speedDown !== null) out.add("homelab_download_speed_bytes", "gauge", "How fast the download client is downloading, in bytes per second.", state.speedDown, { ...labels, direction: "down" });
if (state.speedUp !== null) out.add("homelab_download_speed_bytes", "gauge", "How fast the download client is downloading, in bytes per second.", state.speedUp, { ...labels, direction: "up" });
for (const st of DOWNLOAD_STATES) if (state.counts[st] > 0 || st === "downloading" || st === "queued") out.add("homelab_download_items", "gauge", "How many items the download client has in each state.", state.counts[st], { ...labels, state: st });
out.add("homelab_download_stuck_items", "gauge", "How many items are stuck (in an error state, or stalled with nothing moving for a day).", downloadStuck(state.items, now.getTime()).length, labels);
out.add("homelab_download_paused", "gauge", "1 if the download client's whole queue is paused, else 0.", state.paused ? 1 : 0, labels);
if (state.connection !== null) out.add("homelab_download_connected", "gauge", "1 if the download client can reach the network (not 'disconnected'), else 0.", state.connection === "disconnected" ? 0 : 1, labels);
if (state.failed !== null) out.add("homelab_download_failed_recent", "gauge", "How many downloads failed in the last day.", downloadsFailed(state.failed, now.getTime()).length, labels);
for (const d of state.disks ?? []) out.add("homelab_download_disk_free_bytes", "gauge", "Free space of a folder the download client uses, in bytes.", d.free, { ...labels, folder: d.label });
}
}
/** Each Fragnet game server, for /metrics: whether it is running, its disk and memory against their limits and when it was last backed up. Only while the reading is recent. */
async function addFragnetMetrics(out: Exposition, now: Date) {
const statuses = await fragnetSyncStatuses();
@@ -586,6 +608,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addSpeedtestMetrics(out, now);
await addFragnetMetrics(out, now);
await addArrMetrics(out, now);
await addDownloadMetrics(out, now);
await addAdguardMetrics(out, now);
await addTechnitiumMetrics(out, now);
await addVersionMetrics(out);