Add qBittorrent and SABnzbd: a Download clients page with stuck, disconnected, paused and full-disk alerts

Each client is read every five minutes (read-only; qBittorrent's sign-in is the only POST): speed, queue and item states, network connection or paused queue, free space, SABnzbd's failures and warnings. Shared adapter, sync service, download_state table, /api/downloads, metrics and docs.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-11-04 23:13:03 +01:00
1 parent c7775e5dd9
commit 7618bf0f0a
32 files changed
+10715 -12

No files matched your search

+2
View File
@@ -73,6 +73,7 @@ import { arrRouter } from "./routes/arr.js";
import { wallRouter } from "./routes/wall.js";
import { adguardRouter } from "./routes/adguard.js";
import { technitiumRouter } from "./routes/technitium.js";
import { downloadsRouter } from "./routes/downloads.js";
import { grafanaRouter } from "./routes/grafana.js";
import { versionsRouter } from "./routes/versions.js";
import { portPresetsRouter } from "./routes/portPresets.js";
@@ -227,6 +228,7 @@ export function createApp() {
app.use("/api/wall", wallRouter);
app.use("/api/adguard", adguardRouter);
app.use("/api/technitium", technitiumRouter);
app.use("/api/downloads", downloadsRouter);
app.use("/api/grafana", grafanaRouter);
app.use("/api/versions", versionsRouter);
app.use("/api/port-presets", portPresetsRouter);
+27
View File
@@ -578,6 +578,8 @@ export const integrationTypes = [
"grafana",
"lidarr",
"readarr",
"qbittorrent",
"sabnzbd",
] as const;
export type IntegrationType = (typeof integrationTypes)[number];
@@ -1365,6 +1367,31 @@ export const arrState = sqliteTable("arr_state", {
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── qBittorrent, SABnzbd ───────────────────────────────────────────────────
// How a download client (an integration of type "qbittorrent" or "sabnzbd") is right now, one row each, replaced at every read: how fast it is going, whether it can reach the network (qBittorrent) or
// is paused (SABnzbd), how many items are in each state, the items that are not just sitting finished, what failed lately (SABnzbd), the room its download folders have and its own warnings
// (SABnzbd) — plus since when it has been paused, disconnected or idle with work waiting, which only this app can know. A column that is null could not be read, which is not the same as none.
// Read by services/downloadsSync.ts.
export const downloadState = sqliteTable("download_state", {
integrationId: integer("integration_id")
.primaryKey()
.references(() => integrations.id, { onDelete: "cascade" }),
version: text("version"),
speedDown: integer("speed_down"), // bytes per second; null: not said
speedUp: integer("speed_up"),
paused: integer("paused", { mode: "boolean" }).notNull().default(false),
connection: text("connection"), // connected | firewalled | disconnected; null: not said (always null for SABnzbd)
counts: text("counts").notNull(), // JSON: items in each state
total: integer("total").notNull().default(0),
items: text("items").notNull().default("[]"), // JSON: the items that are not finished and idle, at most 100
failed: text("failed"), // JSON: failed downloads (SABnzbd's history); null: not said
disks: text("disks"), // JSON; null: could not be read
warnings: text("warnings"), // JSON { count, recent }; null: not read
since: text("since").notNull().default("{}"), // JSON { pausedSince, disconnectedSince, idleSince }
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── AdGuard Home ───────────────────────────────────────────────────────────
// How an AdGuard Home (an integration of type "adguard") is right now, one row each, replaced at every read: whether its DNS server runs and protects, the statistics, its filter lists and the result
+2
View File
@@ -44,6 +44,7 @@ import { initFragnetSync } from "./services/fragnetSync.js";
import { initArrSync } from "./services/arrSync.js";
import { initAdguardSync } from "./services/adguardSync.js";
import { initTechnitiumSync } from "./services/technitiumSync.js";
import { initDownloadsSync } from "./services/downloadsSync.js";
import { initGrafanaSync } from "./services/grafanaSync.js";
import { initVersionCheck } from "./services/appVersions.js";
import { backfillMetricRollups } from "./services/serverMetrics.js";
@@ -111,6 +112,7 @@ initFragnetSync();
initArrSync();
initAdguardSync();
initTechnitiumSync();
initDownloadsSync();
initGrafanaSync();
initVersionCheck();
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
@@ -0,0 +1,455 @@
/**
* qBittorrent and SABnzbd adapter — the two download clients, one shape for what they say, so one page, one stored state and one set of alerts serve both.
*
* Reading only: how fast it is downloading and uploading, what is in its queue and in what state, whether it can reach the network (qBittorrent), whether it is paused (SABnzbd), what failed
* lately (SABnzbd's history), and how much room the download folders have. Nothing is sent that adds, pauses, resumes, removes or changes anything — except that qBittorrent has to be signed in to
* (a POST to its login, and a sign-out when the read is done).
*
* qBittorrent (Web API v2, `/api/v2/...`): config url, username and password (optional: a qBittorrent that lets this address in without signing in needs none), insecure. It signs in with the
* user name and password for a session cookie, reads `/sync/maindata` (every torrent, and the free space of its download folder; `/torrents/info` when that is not answered) and
* `/transfer/info`. SABnzbd (`/api?mode=…&output=json`): config url, apiKey (Config → General → Security → API Key — the full key, not the NZB key), insecure. It reads `mode=queue`, `mode=history`
* and `mode=warnings`.
*
* Endpoints and field names are from the two applications' published API documentation and have not been checked against live installs here; every field is read defensively.
*/
import { requestJson } from "../httpJson.js";
import { withDiagLogging } from "../../services/diagLog.js";
export type DownloadKind = "qbittorrent" | "sabnzbd";
export const DOWNLOAD_KINDS: DownloadKind[] = ["qbittorrent", "sabnzbd"];
export const DOWNLOAD_NAMES: Record<DownloadKind, string> = { qbittorrent: "qBittorrent", sabnzbd: "SABnzbd" };
export interface DownloadConfig {
url: string;
username?: string;
password?: string;
apiKey?: string;
insecure?: boolean;
}
/**
* What an item is doing, in words both clients share:
* downloading · queued (waiting its turn) · waiting (cannot start yet: SABnzbd's "propagating") · stalled (qBittorrent: nobody to get it from) · paused · processing (checking, verifying, repairing,
* unpacking, moving) · error · seeding · completed (finished and idle) · failed (SABnzbd's history) · other.
*/
export type DownloadState = "downloading" | "queued" | "waiting" | "stalled" | "paused" | "processing" | "error" | "seeding" | "completed" | "failed" | "other";
export const DOWNLOAD_STATES: DownloadState[] = ["downloading", "queued", "waiting", "stalled", "paused", "processing", "error", "seeding", "completed", "failed", "other"];
export interface DownloadItem {
id: string;
name: string;
state: DownloadState;
/** The client's own word for it ("stalledDL", "Verifying" …). */
rawState: string;
/** 0 to 1; null when not said. */
progress: number | null;
size: number | null;
left: number | null;
/** Bytes per second down; null when not said. */
speed: number | null;
/** Seconds to go; null when not said or unknown. */
eta: number | null;
addedAt: string | null;
/** The last time anything moved (qBittorrent); for a history entry, when it finished. */
activityAt: string | null;
category: string | null;
/** What it says went wrong, when it says. */
message: string | null;
}
export interface DownloadDisk {
label: string;
free: number;
/** Null when the client does not say how big the disk is (qBittorrent says only what is free). */
total: number | null;
}
export type Connection = "connected" | "firewalled" | "disconnected";
export interface DownloadRead {
version: string | null;
/** Bytes per second; null when not said. */
speedDown: number | null;
speedUp: number | null;
/** The whole queue is paused (SABnzbd). */
paused: boolean;
/** Whether it can reach the network (qBittorrent); null when not said. */
connection: Connection | null;
/** How many items there are in each state — all of them, not just the ones listed in `items`. */
counts: Record<DownloadState, number>;
total: number;
/** The items that are not just sitting finished — at most MAX_ITEMS, the ones with something to say first. */
items: DownloadItem[];
/** Downloads that failed lately (SABnzbd's history); null when the client has none to report or it could not be read. */
failed: DownloadItem[] | null;
disks: DownloadDisk[] | null;
/** The client's own list of warnings (SABnzbd); null when not read. */
warnings: { count: number; recent: string[] } | null;
}
export interface DownloadAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
read(): Promise<DownloadRead>;
}
export const MAX_ITEMS = 100;
export const MAX_FAILED = 20;
const GB = 1024 ** 3;
const MB = 1024 ** 2;
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
const arr = (v: unknown): unknown[] => (Array.isArray(v) ? v : []);
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
/** A number, or a string that is one (SABnzbd sends its figures as text). */
const num = (v: unknown): number | null => {
if (typeof v === "number") return Number.isFinite(v) ? v : null;
if (typeof v === "string" && v.trim() !== "" && Number.isFinite(Number(v))) return Number(v);
return null;
};
/** Seconds since 1970 (what both send for a time) as a date; 0 and below mean "never". */
const unixIso = (v: unknown): string | null => {
const n = num(v);
return n !== null && n > 0 ? new Date(n * 1000).toISOString() : null;
};
export const emptyCounts = (): Record<DownloadState, number> => Object.fromEntries(DOWNLOAD_STATES.map((s) => [s, 0])) as Record<DownloadState, number>;
/** How much of the list a page or a judgment looks at first: what needs a look, then what is moving, then what waits. */
const ORDER: Record<DownloadState, number> = { error: 0, failed: 1, stalled: 2, downloading: 3, processing: 4, queued: 5, waiting: 6, paused: 7, other: 8, seeding: 9, completed: 10 };
/** Pure: the counts, and the items worth listing (not the finished and idle ones), the ones with something to say first. */
export function summarise(all: DownloadItem[]): { counts: Record<DownloadState, number>; total: number; items: DownloadItem[] } {
const counts = emptyCounts();
for (const i of all) counts[i.state]++;
const items = all
.filter((i) => i.state !== "seeding" && i.state !== "completed")
.sort((a, b) => ORDER[a.state] - ORDER[b.state] || a.name.localeCompare(b.name))
.slice(0, MAX_ITEMS);
return { counts, total: all.length, items };
}
// ─── qBittorrent ────────────────────────────────────────────────────────────
const QB_STATES: Record<string, DownloadState> = {
error: "error",
missingFiles: "error",
downloading: "downloading",
forcedDL: "downloading",
metaDL: "downloading",
forcedMetaDL: "downloading",
stalledDL: "stalled",
queuedDL: "queued",
pausedDL: "paused",
stoppedDL: "paused",
checkingDL: "processing",
checkingUP: "processing",
checkingResumeData: "processing",
allocating: "processing",
moving: "processing",
uploading: "seeding",
forcedUP: "seeding",
stalledUP: "seeding",
queuedUP: "seeding",
pausedUP: "completed",
stoppedUP: "completed",
};
/** Pure: one torrent of `/torrents/info` (or of `/sync/maindata`, with its hash put in). */
export function parseTorrent(raw: unknown): DownloadItem | null {
const t = obj(raw);
const id = str(t.hash);
if (!id) return null;
const rawState = str(t.state) ?? "unknown";
const progress = num(t.progress);
const eta = num(t.eta);
const size = num(t.size) ?? num(t.total_size);
return {
id,
name: str(t.name) ?? id,
state: QB_STATES[rawState] ?? "other",
rawState,
progress: progress === null ? null : Math.min(1, Math.max(0, progress)),
size,
left: num(t.amount_left),
speed: num(t.dlspeed),
// qBittorrent says 8640000 (a hundred days) for "never".
eta: eta !== null && eta >= 0 && eta < 8_640_000 ? eta : null,
addedAt: unixIso(t.added_on),
activityAt: unixIso(t.last_activity),
category: str(t.category),
message: null,
};
}
/** Pure: `/sync/maindata` — its torrents are an object keyed by hash. */
export function parseMaindataTorrents(raw: unknown): DownloadItem[] {
const out: DownloadItem[] = [];
for (const [hash, t] of Object.entries(obj(obj(raw).torrents))) {
const item = parseTorrent({ hash, ...obj(t) });
if (item) out.push(item);
}
return out;
}
const connectionOf = (v: unknown): Connection | null => {
const s = str(v)?.toLowerCase();
return s === "connected" || s === "firewalled" || s === "disconnected" ? s : null;
};
/** Pure: the numbers qBittorrent keeps about itself (`server_state` of `/sync/maindata`, or `/transfer/info` — the same names). */
export function parseTransfer(raw: unknown): { speedDown: number | null; speedUp: number | null; connection: Connection | null; free: number | null } {
const s = obj(raw);
const free = num(s.free_space_on_disk);
return { speedDown: num(s.dl_info_speed), speedUp: num(s.up_info_speed), connection: connectionOf(s.connection_status), free: free !== null && free >= 0 ? free : null };
}
// ─── SABnzbd ────────────────────────────────────────────────────────────────
const SAB_STATES: Record<string, DownloadState> = {
downloading: "downloading",
queued: "queued",
paused: "paused",
grabbing: "processing",
fetching: "processing",
verifying: "processing",
repairing: "processing",
extracting: "processing",
moving: "processing",
running: "processing",
checking: "processing",
"quick check": "processing",
propagating: "waiting",
completed: "completed",
failed: "failed",
};
/** Pure: one slot of `mode=queue`. Its figures are text, in megabytes. */
export function parseSabSlot(raw: unknown): DownloadItem | null {
const s = obj(raw);
const id = str(s.nzo_id);
if (!id) return null;
const rawState = str(s.status) ?? "unknown";
const mb = num(s.mb);
const mbLeft = num(s.mbleft);
const pct = num(s.percentage);
return {
id,
name: str(s.filename) ?? str(s.name) ?? id,
state: SAB_STATES[rawState.toLowerCase()] ?? "other",
rawState,
progress: pct !== null ? Math.min(1, Math.max(0, pct / 100)) : mb !== null && mb > 0 && mbLeft !== null ? Math.min(1, Math.max(0, (mb - mbLeft) / mb)) : null,
size: mb === null ? null : Math.round(mb * MB),
left: mbLeft === null ? null : Math.round(mbLeft * MB),
speed: null,
eta: parseTimeLeft(s.timeleft),
addedAt: null,
activityAt: null,
category: str(s.cat) && str(s.cat) !== "*" ? str(s.cat) : null,
message: null,
};
}
/** Pure: "0:12:34" or "1:00:12:34" (days first) as seconds; null for anything else, and for "0:00:00" (no estimate). */
export function parseTimeLeft(v: unknown): number | null {
const s = str(v);
if (!s || !/^\d+(:\d+){1,3}$/.test(s)) return null;
const parts = s.split(":").map(Number);
let seconds = 0;
for (const p of parts) seconds = seconds * 60 + p;
if (parts.length === 4) seconds = (((parts[0] * 24 + parts[1]) * 60 + parts[2]) * 60 + parts[3]);
return seconds > 0 ? seconds : null;
}
/** Pure: `mode=queue` — the slots and what the queue says of itself. */
export function parseSabQueue(raw: unknown): { version: string | null; paused: boolean; speedDown: number | null; items: DownloadItem[]; disks: DownloadDisk[] | null } {
const q = obj(obj(raw).queue);
const kbps = num(q.kbpersec);
const disks: DownloadDisk[] = [];
const seen = new Set<string>();
for (const [n, label] of [["1", "Incomplete folder"], ["2", "Complete folder"]] as const) {
const free = num(q[`diskspace${n}`]);
const total = num(q[`diskspacetotal${n}`]);
if (free === null || total === null || total <= 0) continue;
const key = `${total}:${free}`;
if (seen.has(key)) continue; // both folders on one disk: one disk
seen.add(key);
disks.push({ label, free: Math.round(free * GB), total: Math.round(total * GB) });
}
const status = str(q.status)?.toLowerCase();
return {
version: str(q.version),
paused: q.paused === true || q.paused === "1" || status === "paused",
speedDown: kbps === null ? null : Math.round(kbps * 1024),
items: arr(q.slots).map(parseSabSlot).filter((i): i is DownloadItem => i !== null),
disks: disks.length > 0 ? disks : null,
};
}
/** Pure: `mode=history` — what failed (finished downloads are not wanted: they are not news). */
export function parseSabFailed(raw: unknown): DownloadItem[] {
const out: DownloadItem[] = [];
for (const s of arr(obj(obj(raw).history).slots)) {
const h = obj(s);
if (str(h.status)?.toLowerCase() !== "failed") continue;
const id = str(h.nzo_id);
if (!id) continue;
const bytes = num(h.bytes);
out.push({
id,
name: str(h.name) ?? id,
state: "failed",
rawState: "Failed",
progress: null,
size: bytes,
left: null,
speed: null,
eta: null,
addedAt: null,
activityAt: unixIso(h.completed),
category: str(h.category) && str(h.category) !== "*" ? str(h.category) : null,
message: str(h.fail_message),
});
}
return out.sort((a, b) => Date.parse(b.activityAt ?? "") - Date.parse(a.activityAt ?? "")).slice(0, MAX_FAILED);
}
/** Pure: `mode=warnings` — each is a line of text or an object with the text. */
export function parseSabWarnings(raw: unknown): { count: number; recent: string[] } {
const list = arr(obj(raw).warnings)
.map((w) => (typeof w === "string" ? w : str(obj(w).text)))
.map((w) => (w === null ? null : w.replace(/\s+/g, " ").trim()))
.filter((w): w is string => !!w);
return { count: list.length, recent: list.slice(-5).reverse().map((w) => (w.length > 240 ? `${w.slice(0, 237)}…` : w)) };
}
// ─── The adapter ────────────────────────────────────────────────────────────
const soft = async <T>(run: () => Promise<T>): Promise<T | null> => {
try {
return await run();
} catch {
return null;
}
};
export function createDownloadAdapter(kind: DownloadKind, config: DownloadConfig): DownloadAdapter {
const root = String(config.url ?? "").trim().replace(/\/+$/, "");
const insecure = config.insecure === true;
const name = DOWNLOAD_NAMES[kind];
const answer = (res: { status: number; json: unknown; text: string }, what: string): void => {
if (res.status === 404) throw new Error(`${name} has no API at this address (HTTP 404) — check the address (its own, not a proxy's), and that it is ${name}`);
if (res.status < 200 || res.status >= 300) throw new Error(`${name} error: HTTP ${res.status}${what ? ` (${what})` : ""}`);
};
// ── qBittorrent ──
async function withQbSession<T>(run: (get: (path: string) => Promise<{ json: unknown; text: string }>) => Promise<T>): Promise<T> {
const base = `${root}/api/v2`;
// Its protection against forged requests wants the page the request seems to come from to be its own.
const common: Record<string, string> = { Referer: `${root}/`, Origin: root };
let cookie = "";
const username = String(config.username ?? "").trim();
if (username) {
const form = `username=${encodeURIComponent(username)}&password=${encodeURIComponent(String(config.password ?? ""))}`;
const res = await requestJson(`${base}/auth/login`, { method: "POST", rawBody: form, headers: { ...common, "Content-Type": "application/x-www-form-urlencoded" }, insecure });
if (res.status === 403) throw new Error(`${name} refused the sign-in (HTTP 403) — it bans an address for a while after too many wrong passwords; wait, then check the user name and password (Tools → Options → Web UI)`);
if (res.status === 404) throw new Error(`${name} has no Web UI at this address (HTTP 404) — check the address, and that the Web UI is switched on`);
if (res.status !== 200) throw new Error(`${name} sign-in failed: HTTP ${res.status}`);
if (res.text.trim().toLowerCase().startsWith("fail")) throw new Error(`${name} did not accept the user name and password — check them (Tools → Options → Web UI)`);
const set = res.headers["set-cookie"];
const sid = arr(set).map(String).map((c) => /(?:^|;\s*)(SID=[^;]+)/.exec(c)?.[1]).find(Boolean);
if (sid) cookie = sid;
}
const get = async (path: string): Promise<{ json: unknown; text: string }> => {
const res = await requestJson(`${base}${path}`, { headers: { ...common, ...(cookie ? { Cookie: cookie } : {}) }, insecure });
if (res.status === 401 || res.status === 403) throw new Error(`${name} did not let this address in (HTTP ${res.status}) — give the user name and password of its Web UI, or let this address in without signing in (Tools → Options → Web UI → Bypass authentication for clients in whitelisted IP subnets)`);
answer(res, path);
return res;
};
try {
return await run(get);
} finally {
if (cookie) await soft(() => requestJson(`${base}/auth/logout`, { method: "POST", headers: { ...common, Cookie: cookie }, insecure }));
}
}
async function readQb(): Promise<DownloadRead> {
return withQbSession(async (get) => {
const versionText = (await get("/app/version")).text.trim();
const maindata = await soft(async () => (await get("/sync/maindata?rid=0")).json);
let items = maindata && obj(maindata).torrents !== undefined ? parseMaindataTorrents(maindata) : null;
if (items === null) items = arr((await get("/torrents/info")).json).map(parseTorrent).filter((i): i is DownloadItem => i !== null);
const state = obj(obj(maindata).server_state);
const transfer = parseTransfer(Object.keys(state).length > 0 ? state : await soft(async () => (await get("/transfer/info")).json));
const { counts, total, items: listed } = summarise(items);
return {
version: versionText && versionText.length < 40 ? versionText.replace(/^v/i, "") : null,
speedDown: transfer.speedDown,
speedUp: transfer.speedUp,
paused: false,
connection: transfer.connection,
counts,
total,
items: listed,
failed: null,
disks: transfer.free === null ? null : [{ label: "Download folder", free: transfer.free, total: null }],
warnings: null,
};
});
}
// ── SABnzbd ──
async function sab(mode: string, extra = ""): Promise<unknown> {
const key = String(config.apiKey ?? "").trim();
const res = await requestJson(`${root}/api?mode=${mode}&output=json${extra}&apikey=${encodeURIComponent(key)}`, { insecure });
if (res.status === 401 || res.status === 403) throw new Error(`${name} did not accept the API key (HTTP ${res.status}) — copy it again from Config → General → Security → API Key`);
answer(res, mode);
if (res.json === null) throw new Error(`${name} answered with something that isn't JSON — is the address its own?`);
const body = obj(res.json);
// It answers 200 with {"status": false, "error": "..."} when it refuses.
if (body.status === false && typeof body.error === "string") {
throw new Error(/api key/i.test(body.error) ? `${name} did not accept the API key (“${body.error}”) — copy it again from Config → General → Security → API Key (the API key, not the NZB key)` : `${name} said: ${body.error}`);
}
return res.json;
}
async function readSab(): Promise<DownloadRead> {
// The queue must work: it is what says it is there.
const queue = parseSabQueue(await sab("queue", `&limit=${MAX_ITEMS * 2}`));
const [history, warnings] = await Promise.all([soft(() => sab("history", "&limit=50")), soft(() => sab("warnings"))]);
const { counts, total, items } = summarise(queue.items);
return {
version: queue.version,
speedDown: queue.speedDown,
speedUp: null,
paused: queue.paused,
connection: null,
counts,
total,
items,
failed: history === null ? null : parseSabFailed(history),
disks: queue.disks,
warnings: warnings === null ? null : parseSabWarnings(warnings),
};
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
const start = Date.now();
try {
if (kind === "sabnzbd") {
const q = obj(obj(await sab("queue", "&limit=1")).queue);
if (!("slots" in q) && !("status" in q)) throw new Error("That does not look like SABnzbd — is the address its own?");
} else {
await withQbSession(async (get) => {
const v = (await get("/app/version")).text.trim();
if (!/^v?\d+(\.\d+)+/.test(v)) throw new Error("That does not look like qBittorrent — is the address its own?");
});
}
return { ok: true, latencyMs: Date.now() - start };
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) };
}
}
return withDiagLogging(kind, { ping, read: kind === "sabnzbd" ? readSab : readQb });
}
+11
View File
@@ -141,6 +141,17 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "Settings → General → Security → API Key" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
qbittorrent: [
{ key: "url", label: "qBittorrent Web UI URL", secret: false, placeholder: "http://qbittorrent.example.lan:8080" },
{ key: "username", label: "User name (optional)", secret: false, optional: true, placeholder: "empty when qBittorrent lets this address in without signing in" },
{ key: "password", label: "Password (optional)", secret: true, optional: true, type: "password", placeholder: "Tools → Options → Web UI" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
sabnzbd: [
{ key: "url", label: "SABnzbd URL", secret: false, placeholder: "http://sabnzbd.example.lan:8080" },
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "Config → General → Security → API Key (the API key, not the NZB key)" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
adguard: [
{ key: "url", label: "AdGuard Home URL", secret: false, placeholder: "http://adguard.example.lan:3000" },
{ key: "username", label: "User name (optional)", secret: false, optional: true, placeholder: "empty when AdGuard Home has no sign-in" },
+13 -2
View File
@@ -15,7 +15,18 @@ export interface JsonResponse {
const TIMEOUT_MS = 15_000;
export function requestJson(url: string, opts: { method?: "GET" | "POST" | "PUT" | "PATCH" | "DELETE"; headers?: Record<string, string>; body?: unknown; insecure?: boolean; timeoutMs?: number } = {}): Promise<JsonResponse> {
export function requestJson(
url: string,
opts: {
method?: "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
headers?: Record<string, string>;
body?: unknown;
/** A body sent as it is (a form, say), with the Content-Type the caller names in `headers`; used instead of `body`. */
rawBody?: string;
insecure?: boolean;
timeoutMs?: number;
} = {},
): Promise<JsonResponse> {
return new Promise((resolve, reject) => {
let parsed: URL;
try {
@@ -26,7 +37,7 @@ export function requestJson(url: string, opts: { method?: "GET" | "POST" | "PUT"
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return reject(new Error("The address must start with http:// or https://"));
const isHttps = parsed.protocol === "https:";
const lib = isHttps ? https : http;
const payload = opts.body === undefined ? null : JSON.stringify(opts.body);
const payload = opts.rawBody !== undefined ? opts.rawBody : opts.body === undefined ? null : JSON.stringify(opts.body);
const limit = opts.timeoutMs ?? TIMEOUT_MS;
const started = Date.now();
const method = opts.method ?? "GET";
+4
View File
@@ -26,6 +26,7 @@ import { createSpeedtestAdapter } from "./speedtest/adapter.js";
import { createFragnetAdapter } from "./fragnet/adapter.js";
import { createArrAdapter } from "./arr/adapter.js";
import { createAdguardAdapter } from "./adguard/adapter.js";
import { createDownloadAdapter } from "./downloads/adapter.js";
import { createTechnitiumAdapter } from "./technitium/adapter.js";
import { createGrafanaAdapter } from "./grafana/adapter.js";
import { createNextcloudAdapter } from "./nextcloud/adapter.js";
@@ -95,6 +96,9 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
case "lidarr":
case "readarr":
return createArrAdapter(type, config as any);
case "qbittorrent":
case "sabnzbd":
return createDownloadAdapter(type, config as any);
case "adguard":
return createAdguardAdapter(config as any);
case "technitium":
+46
View File
@@ -0,0 +1,46 @@
import { Router } from "express";
import { inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { downloadState, integrations } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { DOWNLOAD_KINDS, type DownloadKind } from "../integrations/downloads/adapter.js";
import { downloadProblems, downloadSyncStatuses, isDownloadFresh, recentFailed, storedOf, stuckItems } from "../services/downloadsSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const downloadsRouter = Router();
downloadsRouter.use(requireAuth);
/** Every qBittorrent and SABnzbd, how it is, from what the last reads stored: speed, what is in its queue and in what state, what is stuck or failed lately, its disk and what is wrong. Everyone signed in can see it. Nothing here changes anything in them. */
downloadsRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, states, statuses] = await Promise.all([
db.select({ id: integrations.id, type: integrations.type, name: integrations.name, enabled: integrations.enabled, baseUrl: integrations.baseUrl }).from(integrations).where(inArray(integrations.type, DOWNLOAD_KINDS)).orderBy(integrations.name),
db.select().from(downloadState),
downloadSyncStatuses(),
]);
const byId = new Map(states.map((s) => [s.integrationId, s]));
const now = Date.now();
res.json({
clients: rows.map((r) => {
const row = byId.get(r.id);
const state = row ? storedOf(row) : null;
const status = statuses[r.id] ?? null;
const fresh = isDownloadFresh(status ?? undefined, now);
return {
id: r.id,
kind: r.type as DownloadKind,
name: r.name,
enabled: r.enabled,
url: r.baseUrl,
status,
fresh,
state: state && {
...state,
/** The ids of the items that are stuck, so the page marks them the way the alert counts them. */
stuck: stuckItems(state.items, now).map((i) => i.id),
failedRecent: recentFailed(state.failed, now).map((i) => i.id),
},
problems: state && fresh ? downloadProblems(r.type as DownloadKind, r.id, r.name, state, now) : [],
};
}),
});
}));
+5
View File
@@ -43,6 +43,8 @@ import { forgetFragnet, syncFragnet } from "../services/fragnetSync.js";
import { forgetArr, syncArr } from "../services/arrSync.js";
import { ARR_KINDS } from "../integrations/arr/adapter.js";
import { forgetAdguard, syncAdguard } from "../services/adguardSync.js";
import { forgetDownload, syncDownload } from "../services/downloadsSync.js";
import { DOWNLOAD_KINDS } from "../integrations/downloads/adapter.js";
import { forgetTechnitium, syncTechnitium } from "../services/technitiumSync.js";
import { forgetGrafana, syncGrafana } from "../services/grafanaSync.js";
import { forgetNextcloud, syncNextcloud } from "../services/nextcloudSync.js";
@@ -160,6 +162,7 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
if (type === "speedtest") void syncSpeedtest(created.id).then(() => invalidateAlertsCache());
if (type === "fragnet") void syncFragnet(created.id).then(() => invalidateAlertsCache());
if ((ARR_KINDS as string[]).includes(type)) void syncArr(created.id).then(() => invalidateAlertsCache());
if ((DOWNLOAD_KINDS as string[]).includes(type)) void syncDownload(created.id).then(() => invalidateAlertsCache());
if (type === "adguard") void syncAdguard(created.id).then(() => invalidateAlertsCache());
if (type === "technitium") void syncTechnitium(created.id).then(() => invalidateAlertsCache());
if (type === "grafana") void syncGrafana(created.id).then(() => invalidateAlertsCache());
@@ -279,6 +282,7 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
if (updated.type === "speedtest" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncSpeedtest(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "fragnet" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncFragnet(updated.id).then(() => invalidateAlertsCache());
if ((ARR_KINDS as string[]).includes(updated.type) && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncArr(updated.id).then(() => invalidateAlertsCache());
if ((DOWNLOAD_KINDS as string[]).includes(updated.type) && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncDownload(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "adguard" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncAdguard(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "technitium" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncTechnitium(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "grafana" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncGrafana(updated.id).then(() => invalidateAlertsCache());
@@ -405,6 +409,7 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
if (existing.type === "speedtest") await forgetSpeedtest(id);
if (existing.type === "fragnet") await forgetFragnet(id);
if ((ARR_KINDS as string[]).includes(existing.type)) await forgetArr(id);
if ((DOWNLOAD_KINDS as string[]).includes(existing.type)) await forgetDownload(id);
if (existing.type === "adguard") await forgetAdguard(id);
if (existing.type === "technitium") await forgetTechnitium(id);
if (existing.type === "grafana") await forgetGrafana(id);
+5 -1
View File
@@ -91,6 +91,8 @@ const SOURCES: Record<string, { label: string; link: string }> = {
lidarr: { label: "Lidarr", link: "/media" },
readarr: { label: "Readarr", link: "/media" },
prowlarr: { label: "Prowlarr", link: "/media" },
qbittorrent: { label: "qBittorrent", link: "/downloads" },
sabnzbd: { label: "SABnzbd", link: "/downloads" },
adguard: { label: "AdGuard Home", link: "/adguard" },
technitium: { label: "Technitium DNS", link: "/technitium" },
versions: { label: "Versions", link: "/versions" },
@@ -147,6 +149,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "grafana", id: Number(m[2]), category: m[1] === "ds" ? "integration" : "monitoring" };
m = /^(sonarr|radarr|prowlarr|lidarr|readarr)-(health|queue|disk):(\d+)/.exec(key);
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
m = /^(qbittorrent|sabnzbd)-(stuck|connection|paused|stalled|disk):(\d+)/.exec(key);
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
if (m) return { type: "fragnet", id: Number(m[2]), category: m[1] === "disk" ? "disk" : m[1] === "backup" ? "backup" : "monitoring" };
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
@@ -301,7 +305,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr|lidarr|readarr|adguard|technitium|grafana):(\d+)(?::(.+))?$/.exec(h);
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr|lidarr|readarr|qbittorrent|sabnzbd|adguard|technitium|grafana):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] === "images" ? " (age of the images)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
+288
View File
@@ -0,0 +1,288 @@
/**
* Keeps what qBittorrent and SABnzbd say up to date: every five minutes each download client is read — how fast it is going, what is in its queue and in what state, whether it can reach the
* network (qBittorrent) or is paused (SABnzbd), what failed lately (SABnzbd), the room in its download folder — and the state is stored. Pages and alerts are built from what is stored, so none of
* them talks to the clients.
*
* What is raised: **downloads that are stuck** — in an error state (qBittorrent: an error, files gone missing), or stalled with nothing moving for a day (one alert for the lot, naming the first few);
* **no connection** — qBittorrent has torrents but has reported no network at all for fifteen minutes (a VPN that dropped); **paused and forgotten** — SABnzbd's queue has been paused for six hours
* with downloads waiting; **downloading nothing** — SABnzbd has work waiting and is not paused, yet nothing has come in for an hour (a news server that does not answer or does not accept the
* login); **a disk nearly full** (under 10% free, critical under 5%; qBittorrent does not say how big the disk is, so there it is under 20 GB, critical under 5 GB). "Since when" is what this
* app saw: the time of the first read that showed it.
*/
import { and, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { downloadState, integrations } from "../db/schema.js";
import { DOWNLOAD_KINDS, DOWNLOAD_NAMES, createDownloadAdapter, emptyCounts, type Connection, type DownloadDisk, type DownloadItem, type DownloadKind, type DownloadRead, type DownloadState } from "../integrations/downloads/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 5 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 20 * 60_000;
const STATUS_FLAG = "downloadsSync";
const MIN = 60_000;
const HOUR = 60 * MIN;
/** A stalled torrent that has had no activity for this long is stuck, not just slow to find a source. */
export const STALLED_AFTER_MS = 24 * HOUR;
/** No network at all for this long is a dropped connection, not a restart. */
export const DISCONNECTED_AFTER_MS = 15 * MIN;
/** A queue paused for this long, with downloads waiting, is forgotten. */
export const PAUSED_AFTER_MS = 6 * HOUR;
/** Work waiting and nothing coming in for this long is a client that cannot get anything. */
export const IDLE_AFTER_MS = HOUR;
/** A failure is news for this long. */
export const FAILED_RECENT_MS = 24 * HOUR;
export const DISK_WARN_FREE = 0.1;
export const DISK_CRITICAL_FREE = 0.05;
/** Where the disk's size is not said: free space under this is a warning, under the second figure critical. */
export const DISK_WARN_BYTES = 20 * 1024 ** 3;
export const DISK_CRITICAL_BYTES = 5 * 1024 ** 3;
/** A disk smaller than this is not a download disk (a tmpfs, a boot partition). */
export const MIN_DISK_BYTES = 2 * 1024 ** 3;
export interface DownloadSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, DownloadSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, DownloadSyncStatus>) : {};
} catch {
return {};
}
}
export async function downloadSyncStatuses(): Promise<Record<number, DownloadSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isDownloadFresh = (status: DownloadSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof downloadState.$inferSelect;
function parse<T>(stored: string | null): T | null {
if (stored === null) return null;
try {
return JSON.parse(stored) as T;
} catch {
return null;
}
}
/** What the app has seen between reads: the time of the first read that showed each of these; null while it is not so. */
export interface DownloadSince {
pausedSince: string | null;
disconnectedSince: string | null;
idleSince: string | null;
}
/** What is stored of one download client, as the read gave it, plus what the app has seen between reads. */
export interface StoredDownload {
version: string | null;
speedDown: number | null;
speedUp: number | null;
paused: boolean;
connection: Connection | null;
counts: Record<DownloadState, number>;
total: number;
items: DownloadItem[];
failed: DownloadItem[] | null;
disks: DownloadDisk[] | null;
warnings: { count: number; recent: string[] } | null;
since: DownloadSince;
updatedAt: string;
}
const NO_SINCE: DownloadSince = { pausedSince: null, disconnectedSince: null, idleSince: null };
export const storedOf = (row: StateRow): StoredDownload => ({
version: row.version,
speedDown: row.speedDown,
speedUp: row.speedUp,
paused: row.paused,
connection: row.connection === "connected" || row.connection === "firewalled" || row.connection === "disconnected" ? row.connection : null,
counts: { ...emptyCounts(), ...(parse<Record<DownloadState, number>>(row.counts) ?? {}) },
total: row.total,
items: parse<DownloadItem[]>(row.items) ?? [],
failed: parse<DownloadItem[]>(row.failed),
disks: parse<DownloadDisk[]>(row.disks),
warnings: parse<{ count: number; recent: string[] }>(row.warnings),
since: { ...NO_SINCE, ...(parse<Partial<DownloadSince>>(row.since) ?? {}) },
updatedAt: row.updatedAt,
});
/** Pure: items that have not finished and are not sitting idle — what is still to be done. */
export const pendingCount = (counts: Record<DownloadState, number>, total: number): number => total - counts.seeding - counts.completed - counts.failed;
/** Pure: work is waiting for a connection to the news server, yet nothing is coming in. */
export const isIdleWithWork = (r: Pick<DownloadRead, "paused" | "speedDown" | "counts">): boolean => !r.paused && r.speedDown === 0 && r.counts.downloading + r.counts.queued > 0;
/** Pure: what the app has seen between reads, carried from the last read. A thing first seen wrong starts its clock now. */
export function carrySince(read: Pick<DownloadRead, "paused" | "connection" | "speedDown" | "counts" | "total">, previous: DownloadSince | null, now: Date): DownloadSince {
const at = now.toISOString();
return {
pausedSince: read.paused ? (previous?.pausedSince ?? at) : null,
disconnectedSince: read.connection === "disconnected" && read.total > 0 ? (previous?.disconnectedSince ?? at) : null,
idleSince: isIdleWithWork(read) ? (previous?.idleSince ?? at) : null,
};
}
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
export async function storeRead(integrationId: number, r: DownloadRead, now: Date = new Date()): Promise<void> {
const [previous] = await db.select().from(downloadState).where(eq(downloadState.integrationId, integrationId)).limit(1);
const since = carrySince(r, previous ? storedOf(previous).since : null, now);
const values = {
version: r.version,
speedDown: r.speedDown === null ? null : Math.round(r.speedDown),
speedUp: r.speedUp === null ? null : Math.round(r.speedUp),
paused: r.paused,
connection: r.connection,
counts: JSON.stringify(r.counts),
total: r.total,
items: JSON.stringify(r.items),
failed: r.failed && JSON.stringify(r.failed),
disks: r.disks && JSON.stringify(r.disks),
warnings: r.warnings && JSON.stringify(r.warnings),
since: JSON.stringify(since),
updatedAt: now.toISOString(),
};
await db.insert(downloadState).values({ integrationId, ...values }).onConflictDoUpdate({ target: downloadState.integrationId, set: values });
}
/** Reads one download client and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncDownload(integrationId: number, now: Date = new Date(), read?: (kind: DownloadKind, config: Record<string, unknown>) => Promise<DownloadRead>): Promise<DownloadSyncStatus> {
let status: DownloadSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
const kind = loaded?.integration.type as DownloadKind | undefined;
if (!loaded || !kind || !DOWNLOAD_KINDS.includes(kind)) throw new Error("not a qBittorrent or SABnzbd integration");
const result = await (read ? read(kind, loaded.config) : createDownloadAdapter(kind, loaded.config as never).read());
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[downloads] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[downloads] couldn't record the result:", err);
}
return status;
}
export async function syncAllDownloads(now: Date = new Date()): Promise<Record<number, DownloadSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(inArray(integrations.type, DOWNLOAD_KINDS), eq(integrations.enabled, true)));
const out: Record<number, DownloadSyncStatus> = {};
for (const r of rows) out[r.id] = await syncDownload(r.id, now);
return out;
}
export async function forgetDownload(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface DownloadProblem {
key: string;
message: string;
severity: "warning" | "critical";
}
const slug = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40);
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / (24 * HOUR)), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
export const sizeText = (bytes: number): string => {
const gb = bytes / 1024 ** 3;
return gb >= 1024 ? `${(gb / 1024).toFixed(1)} TB` : `${gb.toFixed(gb >= 100 ? 0 : 1)} GB`;
};
/** Pure: how long ago anything moved in a stalled item — its last activity, or when it was added; null when neither is known. */
function idleMs(i: DownloadItem, nowMs: number): number | null {
const at = i.activityAt ?? i.addedAt;
return at === null ? null : nowMs - Date.parse(at);
}
/** Pure: whether an item is stuck — in an error state, or stalled with nothing moving for a day. */
export function isStuck(i: DownloadItem, nowMs: number): boolean {
if (i.state === "error") return true;
if (i.state !== "stalled") return false;
const idle = idleMs(i, nowMs);
return idle !== null && idle >= STALLED_AFTER_MS;
}
/** Pure: the stuck items, the ones in error first, then the longest idle. */
export const stuckItems = (items: DownloadItem[], nowMs: number): DownloadItem[] =>
items.filter((i) => isStuck(i, nowMs)).sort((a, b) => Number(b.state === "error") - Number(a.state === "error") || (idleMs(b, nowMs) ?? 0) - (idleMs(a, nowMs) ?? 0));
/** Pure: the downloads that failed in the last day. */
export const recentFailed = (failed: DownloadItem[] | null, nowMs: number): DownloadItem[] => (failed ?? []).filter((f) => f.activityAt !== null && nowMs - Date.parse(f.activityAt) <= FAILED_RECENT_MS);
/** Pure: what is wrong with one download client, from what is stored for it. */
export function downloadProblems(kind: DownloadKind, id: number, name: string, s: StoredDownload, nowMs: number): DownloadProblem[] {
const out: DownloadProblem[] = [];
const app = DOWNLOAD_NAMES[kind];
const stuck = stuckItems(s.items, nowMs);
if (stuck.length > 0) {
const names = stuck.slice(0, 3).map((i) => i.name);
const worst = stuck[0];
const why = worst.state === "error" ? (worst.rawState === "missingFiles" ? "its files are missing" : "it is in an error state") : `nothing has moved for ${ago(idleMs(worst, nowMs) ?? 0)}`;
out.push({ key: `${kind}-stuck:${id}`, severity: "warning", message: `${name}: ${plural(stuck.length, "download")} ${stuck.length === 1 ? "is" : "are"} stuck — ${names.join("; ")}${stuck.length > 3 ? `; and ${stuck.length - 3} more` : ""} (${why})` });
}
if (s.since.disconnectedSince !== null && s.connection === "disconnected" && nowMs - Date.parse(s.since.disconnectedSince) >= DISCONNECTED_AFTER_MS) {
out.push({ key: `${kind}-connection:${id}`, severity: "warning", message: `${name}: ${app} has had no connection to the network for ${ago(nowMs - Date.parse(s.since.disconnectedSince))} — if it is bound to a VPN, is the VPN up?` });
}
const pending = pendingCount(s.counts, s.total);
if (s.paused && pending > 0 && s.since.pausedSince !== null && nowMs - Date.parse(s.since.pausedSince) >= PAUSED_AFTER_MS) {
out.push({ key: `${kind}-paused:${id}`, severity: "warning", message: `${name}: ${app} has been paused for ${ago(nowMs - Date.parse(s.since.pausedSince))} with ${plural(pending, "download")} waiting` });
}
if (!s.paused && s.speedDown === 0 && s.since.idleSince !== null && nowMs - Date.parse(s.since.idleSince) >= IDLE_AFTER_MS) {
const waiting = s.counts.downloading + s.counts.queued;
out.push({
key: `${kind}-stalled:${id}`,
severity: "warning",
message: `${name}: ${app} has downloaded nothing for ${ago(nowMs - Date.parse(s.since.idleSince))} although ${plural(waiting, "download")} ${waiting === 1 ? "is" : "are"} waiting — can it reach its news server?`,
});
}
for (const d of s.disks ?? []) {
if (d.total !== null) {
if (d.total < MIN_DISK_BYTES) continue;
const free = d.free / d.total;
if (free < DISK_WARN_FREE) {
out.push({ key: `${kind}-disk:${id}:${slug(d.label)}`, severity: free < DISK_CRITICAL_FREE ? "critical" : "warning", message: `${name}: the ${d.label.toLowerCase()} of ${app} has ${sizeText(d.free)} free of ${sizeText(d.total)} (${Math.floor(free * 100)}%)` });
}
} else if (d.free < DISK_WARN_BYTES) {
out.push({ key: `${kind}-disk:${id}:${slug(d.label)}`, severity: d.free < DISK_CRITICAL_BYTES ? "critical" : "warning", message: `${name}: the ${d.label.toLowerCase()} of ${app} has only ${sizeText(d.free)} free` });
}
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initDownloadsSync(): void {
if (timer) return;
const pass = () => void syncAllDownloads().catch((err) => console.error("[downloads] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 43_000);
console.log(`[downloads] reading qBittorrent and SABnzbd every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+26 -1
View File
@@ -1,6 +1,6 @@
import { and, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { technitiumState, authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState , adguardState , grafanaState } from "../db/schema.js";
import { downloadState, technitiumState, authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState , adguardState , grafanaState } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
@@ -39,6 +39,8 @@ import { adguardProblems, adguardSyncStatuses, isAdguardFresh, storedOf as adgua
import { isTechnitiumFresh, storedOf as technitiumStoredOf, technitiumProblems, technitiumSyncStatuses, type TechnitiumProblem } from "./technitiumSync.js";
import { grafanaProblems, grafanaSyncStatuses, isGrafanaFresh, storedOf as grafanaStoredOf, type GrafanaProblem } from "./grafanaSync.js";
import { arrProblems, arrSyncStatuses, isArrFresh, storedOf as arrStoredOf, type ArrProblem } from "./arrSync.js";
import { DOWNLOAD_KINDS, type DownloadKind } from "../integrations/downloads/adapter.js";
import { downloadProblems, downloadSyncStatuses, isDownloadFresh, storedOf as downloadStoredOf, type DownloadProblem } from "./downloadsSync.js";
import { fragnetProblems, fragnetSyncStatuses, isFragnetFresh, optionsOf as fragnetOptions, serversOf as fragnetServersOf, type FragnetProblem } from "./fragnetSync.js";
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
import { recordStorageSamples } from "./storageHistory.js";
@@ -140,6 +142,8 @@ export interface HealthSnapshot {
fragnet?: { integrationId: number; problems: FragnetProblem[] }[];
/** What is wrong with each Sonarr, Radarr and Prowlarr (what it says itself, downloads that are stuck, a disk nearly full), only while that reading is recent (services/arrSync.ts). */
arr?: { integrationId: number; kind: ArrKind; problems: ArrProblem[] }[];
/** What is wrong with each qBittorrent and SABnzbd (downloads that are stuck, no connection, a queue left paused or getting nothing, a disk nearly full), only while that reading is recent (services/downloadsSync.ts). */
downloads?: { integrationId: number; kind: DownloadKind; problems: DownloadProblem[] }[];
/** What is wrong with each AdGuard Home (DNS not running or not answering, protection left off, filter lists not updating), only while that reading is recent (services/adguardSync.ts). */
adguard?: { integrationId: number; problems: AdguardProblem[] }[];
/** What is wrong with each Technitium DNS Server (DNS not answering, blocking left off, block lists not updating, zones expired or not syncing), only while that reading is recent (services/technitiumSync.ts). */
@@ -297,6 +301,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
for (const t of snapshot.technitium ?? []) for (const p of t.problems) out.push({ key: p.key, source: `technitium:${t.integrationId}`, message: p.message, severity: p.severity });
for (const g of snapshot.grafana ?? []) for (const p of g.problems) out.push({ key: p.key, source: `grafana:${g.integrationId}`, message: p.message, severity: p.severity });
// qBittorrent and SABnzbd: downloads that are stuck, no connection, a queue left paused or getting nothing, disks nearly full.
for (const d of snapshot.downloads ?? []) for (const p of d.problems) out.push({ key: p.key, source: `${d.kind}:${d.integrationId}`, message: p.message, severity: p.severity });
// Sonarr, Radarr and Prowlarr: what they say is wrong, downloads that are stuck, disks nearly full.
for (const a of snapshot.arr ?? []) for (const p of a.problems) out.push({ key: p.key, source: `${a.kind}:${a.integrationId}`, message: p.message, severity: p.severity });
@@ -686,6 +693,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
console.error("[health] couldn't judge Sonarr, Radarr and Prowlarr:", err instanceof Error ? err.message : err);
}
// qBittorrent and SABnzbd: read by their own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsD = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(and(inArray(integrations.type, DOWNLOAD_KINDS), eq(integrations.enabled, true)));
if (rowsD.length > 0) {
const statuses = await downloadSyncStatuses();
for (const r of rowsD) {
const [state] = await db.select().from(downloadState).where(eq(downloadState.integrationId, r.id)).limit(1);
if (!state || !isDownloadFresh(statuses[r.id], Date.now())) {
held.add(`${r.type}:${r.id}`);
continue;
}
(snapshot.downloads ??= []).push({ integrationId: r.id, kind: r.type as DownloadKind, problems: downloadProblems(r.type as DownloadKind, r.id, r.name, downloadStoredOf(state), Date.now()) });
}
}
} catch (err) {
console.error("[health] couldn't judge qBittorrent and SABnzbd:", err instanceof Error ? err.message : err);
}
// Fragnet game servers: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsF = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "fragnet"), eq(integrations.enabled, true)));
+1 -1
View File
@@ -4,7 +4,7 @@
*/
export const INTEGRATION_GROUPS: { id: string; label: string; types: string[] }[] = [
{ id: "infrastructure", label: "Infrastructure", types: ["proxmox", "synology", "dockhand", "hetzner", "fragnet", "tailscale"] },
{ id: "apps", label: "Apps", types: ["homeassistant", "jellyfin", "sonarr", "radarr", "lidarr", "readarr", "prowlarr", "immich", "paperless", "nextcloud", "authentik"] },
{ id: "apps", label: "Apps", types: ["homeassistant", "jellyfin", "sonarr", "radarr", "lidarr", "readarr", "prowlarr", "qbittorrent", "sabnzbd", "immich", "paperless", "nextcloud", "authentik"] },
{ id: "backups", label: "Backups", types: ["pbs"] },
{ id: "network", label: "Network", types: ["unifi", "npm", "phpipam", "netbox", "adguard", "technitium"] },
{ id: "monitoring", label: "Monitoring", types: ["uptimekuma", "prometheus", "grafana", "ups", "speedtest"] },
+2
View File
@@ -45,6 +45,8 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `integration:${m[1]}`;
m = /^(?:sonarr|radarr|prowlarr|lidarr|readarr)-(?:health|queue|disk):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^(?:qbittorrent|sabnzbd)-(?:stuck|connection|paused|stalled|disk):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^version-behind:(\d+)$/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^adguard-(?:stopped|dns|protection|filters):(\d+)/.exec(key);
+24 -1
View File
@@ -5,7 +5,7 @@
*/
import { eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { technitiumState, assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState, adguardState, grafanaState } from "../db/schema.js";
import { downloadState, technitiumState, assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState, adguardState, grafanaState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { backupState, listAll as listDuplicati } from "./duplicati.js";
@@ -15,6 +15,8 @@ import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf, loadBalancersOf as hetznerLoadBalancersOf, certificatesOf as hetznerCertificatesOf, trafficForecast as hetznerTrafficForecast } from "./hetznerSync.js";
import { ARR_KINDS } from "../integrations/arr/adapter.js";
import { DOWNLOAD_KINDS, DOWNLOAD_STATES } from "../integrations/downloads/adapter.js";
import { downloadSyncStatuses, isDownloadFresh, recentFailed as downloadsFailed, storedOf as downloadStoredOf, stuckItems as downloadStuck } from "./downloadsSync.js";
import { versionRows as appVersionRows } from "./appVersions.js";
import { adguardSyncStatuses, isAdguardFresh, storedOf as adguardStoredOf } from "./adguardSync.js";
import { blockListsOverdueMs, isTechnitiumFresh, storedOf as technitiumStoredOf, technitiumSyncStatuses } from "./technitiumSync.js";
@@ -210,6 +212,26 @@ async function addArrMetrics(out: Exposition, now: Date) {
}
}
/** Each qBittorrent and SABnzbd, for /metrics: how fast it is going, the items in each state and how many are stuck, whether it is connected or paused, what failed in the last day and the free space of its disk. Only while the reading is recent. */
async function addDownloadMetrics(out: Exposition, now: Date) {
const statuses = await downloadSyncStatuses();
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations).where(inArray(integrations.type, DOWNLOAD_KINDS))).map((r) => [r.id, r]));
for (const s of await db.select().from(downloadState)) {
const row = rows.get(s.integrationId);
if (!row || !row.enabled || !isDownloadFresh(statuses[s.integrationId], now.getTime())) continue;
const state = downloadStoredOf(s);
const labels = { client: row.type, name: row.name };
if (state.speedDown !== null) out.add("homelab_download_speed_bytes", "gauge", "How fast the download client is downloading, in bytes per second.", state.speedDown, { ...labels, direction: "down" });
if (state.speedUp !== null) out.add("homelab_download_speed_bytes", "gauge", "How fast the download client is downloading, in bytes per second.", state.speedUp, { ...labels, direction: "up" });
for (const st of DOWNLOAD_STATES) if (state.counts[st] > 0 || st === "downloading" || st === "queued") out.add("homelab_download_items", "gauge", "How many items the download client has in each state.", state.counts[st], { ...labels, state: st });
out.add("homelab_download_stuck_items", "gauge", "How many items are stuck (in an error state, or stalled with nothing moving for a day).", downloadStuck(state.items, now.getTime()).length, labels);
out.add("homelab_download_paused", "gauge", "1 if the download client's whole queue is paused, else 0.", state.paused ? 1 : 0, labels);
if (state.connection !== null) out.add("homelab_download_connected", "gauge", "1 if the download client can reach the network (not 'disconnected'), else 0.", state.connection === "disconnected" ? 0 : 1, labels);
if (state.failed !== null) out.add("homelab_download_failed_recent", "gauge", "How many downloads failed in the last day.", downloadsFailed(state.failed, now.getTime()).length, labels);
for (const d of state.disks ?? []) out.add("homelab_download_disk_free_bytes", "gauge", "Free space of a folder the download client uses, in bytes.", d.free, { ...labels, folder: d.label });
}
}
/** Each Fragnet game server, for /metrics: whether it is running, its disk and memory against their limits and when it was last backed up. Only while the reading is recent. */
async function addFragnetMetrics(out: Exposition, now: Date) {
const statuses = await fragnetSyncStatuses();
@@ -586,6 +608,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addSpeedtestMetrics(out, now);
await addFragnetMetrics(out, now);
await addArrMetrics(out, now);
await addDownloadMetrics(out, now);
await addAdguardMetrics(out, now);
await addTechnitiumMetrics(out, now);
await addVersionMetrics(out);