Add qBittorrent and SABnzbd: a Download clients page with stuck, disconnected, paused and full-disk alerts
Each client is read every five minutes (read-only; qBittorrent's sign-in is the only POST): speed, queue and item states, network connection or paused queue, free space, SABnzbd's failures and warnings. Shared adapter, sync service, download_state table, /api/downloads, metrics and docs. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c7775e5dd9
commit
7618bf0f0a
32 files changed
+10715
-12
No files matched your search
@@ -0,0 +1,17 @@
|
||||
CREATE TABLE `download_state` (
|
||||
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||
`version` text,
|
||||
`speed_down` integer,
|
||||
`speed_up` integer,
|
||||
`paused` integer DEFAULT false NOT NULL,
|
||||
`connection` text,
|
||||
`counts` text NOT NULL,
|
||||
`total` integer DEFAULT 0 NOT NULL,
|
||||
`items` text DEFAULT '[]' NOT NULL,
|
||||
`failed` text,
|
||||
`disks` text,
|
||||
`warnings` text,
|
||||
`since` text DEFAULT '{}' NOT NULL,
|
||||
`updated_at` text NOT NULL,
|
||||
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -582,6 +582,13 @@
|
||||
"when": 1793571807053,
|
||||
"tag": "0082_charming_dark_beast",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 83,
|
||||
"version": "6",
|
||||
"when": 1793829376092,
|
||||
"tag": "0083_lovely_midnight",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -73,6 +73,7 @@ import { arrRouter } from "./routes/arr.js";
|
||||
import { wallRouter } from "./routes/wall.js";
|
||||
import { adguardRouter } from "./routes/adguard.js";
|
||||
import { technitiumRouter } from "./routes/technitium.js";
|
||||
import { downloadsRouter } from "./routes/downloads.js";
|
||||
import { grafanaRouter } from "./routes/grafana.js";
|
||||
import { versionsRouter } from "./routes/versions.js";
|
||||
import { portPresetsRouter } from "./routes/portPresets.js";
|
||||
@@ -227,6 +228,7 @@ export function createApp() {
|
||||
app.use("/api/wall", wallRouter);
|
||||
app.use("/api/adguard", adguardRouter);
|
||||
app.use("/api/technitium", technitiumRouter);
|
||||
app.use("/api/downloads", downloadsRouter);
|
||||
app.use("/api/grafana", grafanaRouter);
|
||||
app.use("/api/versions", versionsRouter);
|
||||
app.use("/api/port-presets", portPresetsRouter);
|
||||
|
||||
@@ -578,6 +578,8 @@ export const integrationTypes = [
|
||||
"grafana",
|
||||
"lidarr",
|
||||
"readarr",
|
||||
"qbittorrent",
|
||||
"sabnzbd",
|
||||
] as const;
|
||||
export type IntegrationType = (typeof integrationTypes)[number];
|
||||
|
||||
@@ -1365,6 +1367,31 @@ export const arrState = sqliteTable("arr_state", {
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── qBittorrent, SABnzbd ───────────────────────────────────────────────────
|
||||
|
||||
// How a download client (an integration of type "qbittorrent" or "sabnzbd") is right now, one row each, replaced at every read: how fast it is going, whether it can reach the network (qBittorrent) or
|
||||
// is paused (SABnzbd), how many items are in each state, the items that are not just sitting finished, what failed lately (SABnzbd), the room its download folders have and its own warnings
|
||||
// (SABnzbd) — plus since when it has been paused, disconnected or idle with work waiting, which only this app can know. A column that is null could not be read, which is not the same as none.
|
||||
// Read by services/downloadsSync.ts.
|
||||
export const downloadState = sqliteTable("download_state", {
|
||||
integrationId: integer("integration_id")
|
||||
.primaryKey()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
version: text("version"),
|
||||
speedDown: integer("speed_down"), // bytes per second; null: not said
|
||||
speedUp: integer("speed_up"),
|
||||
paused: integer("paused", { mode: "boolean" }).notNull().default(false),
|
||||
connection: text("connection"), // connected | firewalled | disconnected; null: not said (always null for SABnzbd)
|
||||
counts: text("counts").notNull(), // JSON: items in each state
|
||||
total: integer("total").notNull().default(0),
|
||||
items: text("items").notNull().default("[]"), // JSON: the items that are not finished and idle, at most 100
|
||||
failed: text("failed"), // JSON: failed downloads (SABnzbd's history); null: not said
|
||||
disks: text("disks"), // JSON; null: could not be read
|
||||
warnings: text("warnings"), // JSON { count, recent }; null: not read
|
||||
since: text("since").notNull().default("{}"), // JSON { pausedSince, disconnectedSince, idleSince }
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── AdGuard Home ───────────────────────────────────────────────────────────
|
||||
|
||||
// How an AdGuard Home (an integration of type "adguard") is right now, one row each, replaced at every read: whether its DNS server runs and protects, the statistics, its filter lists and the result
|
||||
|
||||
@@ -44,6 +44,7 @@ import { initFragnetSync } from "./services/fragnetSync.js";
|
||||
import { initArrSync } from "./services/arrSync.js";
|
||||
import { initAdguardSync } from "./services/adguardSync.js";
|
||||
import { initTechnitiumSync } from "./services/technitiumSync.js";
|
||||
import { initDownloadsSync } from "./services/downloadsSync.js";
|
||||
import { initGrafanaSync } from "./services/grafanaSync.js";
|
||||
import { initVersionCheck } from "./services/appVersions.js";
|
||||
import { backfillMetricRollups } from "./services/serverMetrics.js";
|
||||
@@ -111,6 +112,7 @@ initFragnetSync();
|
||||
initArrSync();
|
||||
initAdguardSync();
|
||||
initTechnitiumSync();
|
||||
initDownloadsSync();
|
||||
initGrafanaSync();
|
||||
initVersionCheck();
|
||||
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
|
||||
|
||||
@@ -0,0 +1,455 @@
|
||||
/**
|
||||
* qBittorrent and SABnzbd adapter — the two download clients, one shape for what they say, so one page, one stored state and one set of alerts serve both.
|
||||
*
|
||||
* Reading only: how fast it is downloading and uploading, what is in its queue and in what state, whether it can reach the network (qBittorrent), whether it is paused (SABnzbd), what failed
|
||||
* lately (SABnzbd's history), and how much room the download folders have. Nothing is sent that adds, pauses, resumes, removes or changes anything — except that qBittorrent has to be signed in to
|
||||
* (a POST to its login, and a sign-out when the read is done).
|
||||
*
|
||||
* qBittorrent (Web API v2, `/api/v2/...`): config url, username and password (optional: a qBittorrent that lets this address in without signing in needs none), insecure. It signs in with the
|
||||
* user name and password for a session cookie, reads `/sync/maindata` (every torrent, and the free space of its download folder; `/torrents/info` when that is not answered) and
|
||||
* `/transfer/info`. SABnzbd (`/api?mode=…&output=json`): config url, apiKey (Config → General → Security → API Key — the full key, not the NZB key), insecure. It reads `mode=queue`, `mode=history`
|
||||
* and `mode=warnings`.
|
||||
*
|
||||
* Endpoints and field names are from the two applications' published API documentation and have not been checked against live installs here; every field is read defensively.
|
||||
*/
|
||||
import { requestJson } from "../httpJson.js";
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
|
||||
export type DownloadKind = "qbittorrent" | "sabnzbd";
|
||||
export const DOWNLOAD_KINDS: DownloadKind[] = ["qbittorrent", "sabnzbd"];
|
||||
export const DOWNLOAD_NAMES: Record<DownloadKind, string> = { qbittorrent: "qBittorrent", sabnzbd: "SABnzbd" };
|
||||
|
||||
export interface DownloadConfig {
|
||||
url: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
apiKey?: string;
|
||||
insecure?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* What an item is doing, in words both clients share:
|
||||
* downloading · queued (waiting its turn) · waiting (cannot start yet: SABnzbd's "propagating") · stalled (qBittorrent: nobody to get it from) · paused · processing (checking, verifying, repairing,
|
||||
* unpacking, moving) · error · seeding · completed (finished and idle) · failed (SABnzbd's history) · other.
|
||||
*/
|
||||
export type DownloadState = "downloading" | "queued" | "waiting" | "stalled" | "paused" | "processing" | "error" | "seeding" | "completed" | "failed" | "other";
|
||||
export const DOWNLOAD_STATES: DownloadState[] = ["downloading", "queued", "waiting", "stalled", "paused", "processing", "error", "seeding", "completed", "failed", "other"];
|
||||
|
||||
export interface DownloadItem {
|
||||
id: string;
|
||||
name: string;
|
||||
state: DownloadState;
|
||||
/** The client's own word for it ("stalledDL", "Verifying" …). */
|
||||
rawState: string;
|
||||
/** 0 to 1; null when not said. */
|
||||
progress: number | null;
|
||||
size: number | null;
|
||||
left: number | null;
|
||||
/** Bytes per second down; null when not said. */
|
||||
speed: number | null;
|
||||
/** Seconds to go; null when not said or unknown. */
|
||||
eta: number | null;
|
||||
addedAt: string | null;
|
||||
/** The last time anything moved (qBittorrent); for a history entry, when it finished. */
|
||||
activityAt: string | null;
|
||||
category: string | null;
|
||||
/** What it says went wrong, when it says. */
|
||||
message: string | null;
|
||||
}
|
||||
|
||||
export interface DownloadDisk {
|
||||
label: string;
|
||||
free: number;
|
||||
/** Null when the client does not say how big the disk is (qBittorrent says only what is free). */
|
||||
total: number | null;
|
||||
}
|
||||
|
||||
export type Connection = "connected" | "firewalled" | "disconnected";
|
||||
|
||||
export interface DownloadRead {
|
||||
version: string | null;
|
||||
/** Bytes per second; null when not said. */
|
||||
speedDown: number | null;
|
||||
speedUp: number | null;
|
||||
/** The whole queue is paused (SABnzbd). */
|
||||
paused: boolean;
|
||||
/** Whether it can reach the network (qBittorrent); null when not said. */
|
||||
connection: Connection | null;
|
||||
/** How many items there are in each state — all of them, not just the ones listed in `items`. */
|
||||
counts: Record<DownloadState, number>;
|
||||
total: number;
|
||||
/** The items that are not just sitting finished — at most MAX_ITEMS, the ones with something to say first. */
|
||||
items: DownloadItem[];
|
||||
/** Downloads that failed lately (SABnzbd's history); null when the client has none to report or it could not be read. */
|
||||
failed: DownloadItem[] | null;
|
||||
disks: DownloadDisk[] | null;
|
||||
/** The client's own list of warnings (SABnzbd); null when not read. */
|
||||
warnings: { count: number; recent: string[] } | null;
|
||||
}
|
||||
|
||||
export interface DownloadAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(): Promise<DownloadRead>;
|
||||
}
|
||||
|
||||
export const MAX_ITEMS = 100;
|
||||
export const MAX_FAILED = 20;
|
||||
const GB = 1024 ** 3;
|
||||
const MB = 1024 ** 2;
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
|
||||
const arr = (v: unknown): unknown[] => (Array.isArray(v) ? v : []);
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
|
||||
/** A number, or a string that is one (SABnzbd sends its figures as text). */
|
||||
const num = (v: unknown): number | null => {
|
||||
if (typeof v === "number") return Number.isFinite(v) ? v : null;
|
||||
if (typeof v === "string" && v.trim() !== "" && Number.isFinite(Number(v))) return Number(v);
|
||||
return null;
|
||||
};
|
||||
/** Seconds since 1970 (what both send for a time) as a date; 0 and below mean "never". */
|
||||
const unixIso = (v: unknown): string | null => {
|
||||
const n = num(v);
|
||||
return n !== null && n > 0 ? new Date(n * 1000).toISOString() : null;
|
||||
};
|
||||
|
||||
export const emptyCounts = (): Record<DownloadState, number> => Object.fromEntries(DOWNLOAD_STATES.map((s) => [s, 0])) as Record<DownloadState, number>;
|
||||
|
||||
/** How much of the list a page or a judgment looks at first: what needs a look, then what is moving, then what waits. */
|
||||
const ORDER: Record<DownloadState, number> = { error: 0, failed: 1, stalled: 2, downloading: 3, processing: 4, queued: 5, waiting: 6, paused: 7, other: 8, seeding: 9, completed: 10 };
|
||||
|
||||
/** Pure: the counts, and the items worth listing (not the finished and idle ones), the ones with something to say first. */
|
||||
export function summarise(all: DownloadItem[]): { counts: Record<DownloadState, number>; total: number; items: DownloadItem[] } {
|
||||
const counts = emptyCounts();
|
||||
for (const i of all) counts[i.state]++;
|
||||
const items = all
|
||||
.filter((i) => i.state !== "seeding" && i.state !== "completed")
|
||||
.sort((a, b) => ORDER[a.state] - ORDER[b.state] || a.name.localeCompare(b.name))
|
||||
.slice(0, MAX_ITEMS);
|
||||
return { counts, total: all.length, items };
|
||||
}
|
||||
|
||||
// ─── qBittorrent ────────────────────────────────────────────────────────────
|
||||
|
||||
const QB_STATES: Record<string, DownloadState> = {
|
||||
error: "error",
|
||||
missingFiles: "error",
|
||||
downloading: "downloading",
|
||||
forcedDL: "downloading",
|
||||
metaDL: "downloading",
|
||||
forcedMetaDL: "downloading",
|
||||
stalledDL: "stalled",
|
||||
queuedDL: "queued",
|
||||
pausedDL: "paused",
|
||||
stoppedDL: "paused",
|
||||
checkingDL: "processing",
|
||||
checkingUP: "processing",
|
||||
checkingResumeData: "processing",
|
||||
allocating: "processing",
|
||||
moving: "processing",
|
||||
uploading: "seeding",
|
||||
forcedUP: "seeding",
|
||||
stalledUP: "seeding",
|
||||
queuedUP: "seeding",
|
||||
pausedUP: "completed",
|
||||
stoppedUP: "completed",
|
||||
};
|
||||
|
||||
/** Pure: one torrent of `/torrents/info` (or of `/sync/maindata`, with its hash put in). */
|
||||
export function parseTorrent(raw: unknown): DownloadItem | null {
|
||||
const t = obj(raw);
|
||||
const id = str(t.hash);
|
||||
if (!id) return null;
|
||||
const rawState = str(t.state) ?? "unknown";
|
||||
const progress = num(t.progress);
|
||||
const eta = num(t.eta);
|
||||
const size = num(t.size) ?? num(t.total_size);
|
||||
return {
|
||||
id,
|
||||
name: str(t.name) ?? id,
|
||||
state: QB_STATES[rawState] ?? "other",
|
||||
rawState,
|
||||
progress: progress === null ? null : Math.min(1, Math.max(0, progress)),
|
||||
size,
|
||||
left: num(t.amount_left),
|
||||
speed: num(t.dlspeed),
|
||||
// qBittorrent says 8640000 (a hundred days) for "never".
|
||||
eta: eta !== null && eta >= 0 && eta < 8_640_000 ? eta : null,
|
||||
addedAt: unixIso(t.added_on),
|
||||
activityAt: unixIso(t.last_activity),
|
||||
category: str(t.category),
|
||||
message: null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Pure: `/sync/maindata` — its torrents are an object keyed by hash. */
|
||||
export function parseMaindataTorrents(raw: unknown): DownloadItem[] {
|
||||
const out: DownloadItem[] = [];
|
||||
for (const [hash, t] of Object.entries(obj(obj(raw).torrents))) {
|
||||
const item = parseTorrent({ hash, ...obj(t) });
|
||||
if (item) out.push(item);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const connectionOf = (v: unknown): Connection | null => {
|
||||
const s = str(v)?.toLowerCase();
|
||||
return s === "connected" || s === "firewalled" || s === "disconnected" ? s : null;
|
||||
};
|
||||
|
||||
/** Pure: the numbers qBittorrent keeps about itself (`server_state` of `/sync/maindata`, or `/transfer/info` — the same names). */
|
||||
export function parseTransfer(raw: unknown): { speedDown: number | null; speedUp: number | null; connection: Connection | null; free: number | null } {
|
||||
const s = obj(raw);
|
||||
const free = num(s.free_space_on_disk);
|
||||
return { speedDown: num(s.dl_info_speed), speedUp: num(s.up_info_speed), connection: connectionOf(s.connection_status), free: free !== null && free >= 0 ? free : null };
|
||||
}
|
||||
|
||||
// ─── SABnzbd ────────────────────────────────────────────────────────────────
|
||||
|
||||
const SAB_STATES: Record<string, DownloadState> = {
|
||||
downloading: "downloading",
|
||||
queued: "queued",
|
||||
paused: "paused",
|
||||
grabbing: "processing",
|
||||
fetching: "processing",
|
||||
verifying: "processing",
|
||||
repairing: "processing",
|
||||
extracting: "processing",
|
||||
moving: "processing",
|
||||
running: "processing",
|
||||
checking: "processing",
|
||||
"quick check": "processing",
|
||||
propagating: "waiting",
|
||||
completed: "completed",
|
||||
failed: "failed",
|
||||
};
|
||||
|
||||
/** Pure: one slot of `mode=queue`. Its figures are text, in megabytes. */
|
||||
export function parseSabSlot(raw: unknown): DownloadItem | null {
|
||||
const s = obj(raw);
|
||||
const id = str(s.nzo_id);
|
||||
if (!id) return null;
|
||||
const rawState = str(s.status) ?? "unknown";
|
||||
const mb = num(s.mb);
|
||||
const mbLeft = num(s.mbleft);
|
||||
const pct = num(s.percentage);
|
||||
return {
|
||||
id,
|
||||
name: str(s.filename) ?? str(s.name) ?? id,
|
||||
state: SAB_STATES[rawState.toLowerCase()] ?? "other",
|
||||
rawState,
|
||||
progress: pct !== null ? Math.min(1, Math.max(0, pct / 100)) : mb !== null && mb > 0 && mbLeft !== null ? Math.min(1, Math.max(0, (mb - mbLeft) / mb)) : null,
|
||||
size: mb === null ? null : Math.round(mb * MB),
|
||||
left: mbLeft === null ? null : Math.round(mbLeft * MB),
|
||||
speed: null,
|
||||
eta: parseTimeLeft(s.timeleft),
|
||||
addedAt: null,
|
||||
activityAt: null,
|
||||
category: str(s.cat) && str(s.cat) !== "*" ? str(s.cat) : null,
|
||||
message: null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Pure: "0:12:34" or "1:00:12:34" (days first) as seconds; null for anything else, and for "0:00:00" (no estimate). */
|
||||
export function parseTimeLeft(v: unknown): number | null {
|
||||
const s = str(v);
|
||||
if (!s || !/^\d+(:\d+){1,3}$/.test(s)) return null;
|
||||
const parts = s.split(":").map(Number);
|
||||
let seconds = 0;
|
||||
for (const p of parts) seconds = seconds * 60 + p;
|
||||
if (parts.length === 4) seconds = (((parts[0] * 24 + parts[1]) * 60 + parts[2]) * 60 + parts[3]);
|
||||
return seconds > 0 ? seconds : null;
|
||||
}
|
||||
|
||||
/** Pure: `mode=queue` — the slots and what the queue says of itself. */
|
||||
export function parseSabQueue(raw: unknown): { version: string | null; paused: boolean; speedDown: number | null; items: DownloadItem[]; disks: DownloadDisk[] | null } {
|
||||
const q = obj(obj(raw).queue);
|
||||
const kbps = num(q.kbpersec);
|
||||
const disks: DownloadDisk[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const [n, label] of [["1", "Incomplete folder"], ["2", "Complete folder"]] as const) {
|
||||
const free = num(q[`diskspace${n}`]);
|
||||
const total = num(q[`diskspacetotal${n}`]);
|
||||
if (free === null || total === null || total <= 0) continue;
|
||||
const key = `${total}:${free}`;
|
||||
if (seen.has(key)) continue; // both folders on one disk: one disk
|
||||
seen.add(key);
|
||||
disks.push({ label, free: Math.round(free * GB), total: Math.round(total * GB) });
|
||||
}
|
||||
const status = str(q.status)?.toLowerCase();
|
||||
return {
|
||||
version: str(q.version),
|
||||
paused: q.paused === true || q.paused === "1" || status === "paused",
|
||||
speedDown: kbps === null ? null : Math.round(kbps * 1024),
|
||||
items: arr(q.slots).map(parseSabSlot).filter((i): i is DownloadItem => i !== null),
|
||||
disks: disks.length > 0 ? disks : null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Pure: `mode=history` — what failed (finished downloads are not wanted: they are not news). */
|
||||
export function parseSabFailed(raw: unknown): DownloadItem[] {
|
||||
const out: DownloadItem[] = [];
|
||||
for (const s of arr(obj(obj(raw).history).slots)) {
|
||||
const h = obj(s);
|
||||
if (str(h.status)?.toLowerCase() !== "failed") continue;
|
||||
const id = str(h.nzo_id);
|
||||
if (!id) continue;
|
||||
const bytes = num(h.bytes);
|
||||
out.push({
|
||||
id,
|
||||
name: str(h.name) ?? id,
|
||||
state: "failed",
|
||||
rawState: "Failed",
|
||||
progress: null,
|
||||
size: bytes,
|
||||
left: null,
|
||||
speed: null,
|
||||
eta: null,
|
||||
addedAt: null,
|
||||
activityAt: unixIso(h.completed),
|
||||
category: str(h.category) && str(h.category) !== "*" ? str(h.category) : null,
|
||||
message: str(h.fail_message),
|
||||
});
|
||||
}
|
||||
return out.sort((a, b) => Date.parse(b.activityAt ?? "") - Date.parse(a.activityAt ?? "")).slice(0, MAX_FAILED);
|
||||
}
|
||||
|
||||
/** Pure: `mode=warnings` — each is a line of text or an object with the text. */
|
||||
export function parseSabWarnings(raw: unknown): { count: number; recent: string[] } {
|
||||
const list = arr(obj(raw).warnings)
|
||||
.map((w) => (typeof w === "string" ? w : str(obj(w).text)))
|
||||
.map((w) => (w === null ? null : w.replace(/\s+/g, " ").trim()))
|
||||
.filter((w): w is string => !!w);
|
||||
return { count: list.length, recent: list.slice(-5).reverse().map((w) => (w.length > 240 ? `${w.slice(0, 237)}…` : w)) };
|
||||
}
|
||||
|
||||
// ─── The adapter ────────────────────────────────────────────────────────────
|
||||
|
||||
const soft = async <T>(run: () => Promise<T>): Promise<T | null> => {
|
||||
try {
|
||||
return await run();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
export function createDownloadAdapter(kind: DownloadKind, config: DownloadConfig): DownloadAdapter {
|
||||
const root = String(config.url ?? "").trim().replace(/\/+$/, "");
|
||||
const insecure = config.insecure === true;
|
||||
const name = DOWNLOAD_NAMES[kind];
|
||||
|
||||
const answer = (res: { status: number; json: unknown; text: string }, what: string): void => {
|
||||
if (res.status === 404) throw new Error(`${name} has no API at this address (HTTP 404) — check the address (its own, not a proxy's), and that it is ${name}`);
|
||||
if (res.status < 200 || res.status >= 300) throw new Error(`${name} error: HTTP ${res.status}${what ? ` (${what})` : ""}`);
|
||||
};
|
||||
|
||||
// ── qBittorrent ──
|
||||
async function withQbSession<T>(run: (get: (path: string) => Promise<{ json: unknown; text: string }>) => Promise<T>): Promise<T> {
|
||||
const base = `${root}/api/v2`;
|
||||
// Its protection against forged requests wants the page the request seems to come from to be its own.
|
||||
const common: Record<string, string> = { Referer: `${root}/`, Origin: root };
|
||||
let cookie = "";
|
||||
const username = String(config.username ?? "").trim();
|
||||
if (username) {
|
||||
const form = `username=${encodeURIComponent(username)}&password=${encodeURIComponent(String(config.password ?? ""))}`;
|
||||
const res = await requestJson(`${base}/auth/login`, { method: "POST", rawBody: form, headers: { ...common, "Content-Type": "application/x-www-form-urlencoded" }, insecure });
|
||||
if (res.status === 403) throw new Error(`${name} refused the sign-in (HTTP 403) — it bans an address for a while after too many wrong passwords; wait, then check the user name and password (Tools → Options → Web UI)`);
|
||||
if (res.status === 404) throw new Error(`${name} has no Web UI at this address (HTTP 404) — check the address, and that the Web UI is switched on`);
|
||||
if (res.status !== 200) throw new Error(`${name} sign-in failed: HTTP ${res.status}`);
|
||||
if (res.text.trim().toLowerCase().startsWith("fail")) throw new Error(`${name} did not accept the user name and password — check them (Tools → Options → Web UI)`);
|
||||
const set = res.headers["set-cookie"];
|
||||
const sid = arr(set).map(String).map((c) => /(?:^|;\s*)(SID=[^;]+)/.exec(c)?.[1]).find(Boolean);
|
||||
if (sid) cookie = sid;
|
||||
}
|
||||
const get = async (path: string): Promise<{ json: unknown; text: string }> => {
|
||||
const res = await requestJson(`${base}${path}`, { headers: { ...common, ...(cookie ? { Cookie: cookie } : {}) }, insecure });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`${name} did not let this address in (HTTP ${res.status}) — give the user name and password of its Web UI, or let this address in without signing in (Tools → Options → Web UI → Bypass authentication for clients in whitelisted IP subnets)`);
|
||||
answer(res, path);
|
||||
return res;
|
||||
};
|
||||
try {
|
||||
return await run(get);
|
||||
} finally {
|
||||
if (cookie) await soft(() => requestJson(`${base}/auth/logout`, { method: "POST", headers: { ...common, Cookie: cookie }, insecure }));
|
||||
}
|
||||
}
|
||||
|
||||
async function readQb(): Promise<DownloadRead> {
|
||||
return withQbSession(async (get) => {
|
||||
const versionText = (await get("/app/version")).text.trim();
|
||||
const maindata = await soft(async () => (await get("/sync/maindata?rid=0")).json);
|
||||
let items = maindata && obj(maindata).torrents !== undefined ? parseMaindataTorrents(maindata) : null;
|
||||
if (items === null) items = arr((await get("/torrents/info")).json).map(parseTorrent).filter((i): i is DownloadItem => i !== null);
|
||||
const state = obj(obj(maindata).server_state);
|
||||
const transfer = parseTransfer(Object.keys(state).length > 0 ? state : await soft(async () => (await get("/transfer/info")).json));
|
||||
const { counts, total, items: listed } = summarise(items);
|
||||
return {
|
||||
version: versionText && versionText.length < 40 ? versionText.replace(/^v/i, "") : null,
|
||||
speedDown: transfer.speedDown,
|
||||
speedUp: transfer.speedUp,
|
||||
paused: false,
|
||||
connection: transfer.connection,
|
||||
counts,
|
||||
total,
|
||||
items: listed,
|
||||
failed: null,
|
||||
disks: transfer.free === null ? null : [{ label: "Download folder", free: transfer.free, total: null }],
|
||||
warnings: null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// ── SABnzbd ──
|
||||
async function sab(mode: string, extra = ""): Promise<unknown> {
|
||||
const key = String(config.apiKey ?? "").trim();
|
||||
const res = await requestJson(`${root}/api?mode=${mode}&output=json${extra}&apikey=${encodeURIComponent(key)}`, { insecure });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`${name} did not accept the API key (HTTP ${res.status}) — copy it again from Config → General → Security → API Key`);
|
||||
answer(res, mode);
|
||||
if (res.json === null) throw new Error(`${name} answered with something that isn't JSON — is the address its own?`);
|
||||
const body = obj(res.json);
|
||||
// It answers 200 with {"status": false, "error": "..."} when it refuses.
|
||||
if (body.status === false && typeof body.error === "string") {
|
||||
throw new Error(/api key/i.test(body.error) ? `${name} did not accept the API key (“${body.error}”) — copy it again from Config → General → Security → API Key (the API key, not the NZB key)` : `${name} said: ${body.error}`);
|
||||
}
|
||||
return res.json;
|
||||
}
|
||||
|
||||
async function readSab(): Promise<DownloadRead> {
|
||||
// The queue must work: it is what says it is there.
|
||||
const queue = parseSabQueue(await sab("queue", `&limit=${MAX_ITEMS * 2}`));
|
||||
const [history, warnings] = await Promise.all([soft(() => sab("history", "&limit=50")), soft(() => sab("warnings"))]);
|
||||
const { counts, total, items } = summarise(queue.items);
|
||||
return {
|
||||
version: queue.version,
|
||||
speedDown: queue.speedDown,
|
||||
speedUp: null,
|
||||
paused: queue.paused,
|
||||
connection: null,
|
||||
counts,
|
||||
total,
|
||||
items,
|
||||
failed: history === null ? null : parseSabFailed(history),
|
||||
disks: queue.disks,
|
||||
warnings: warnings === null ? null : parseSabWarnings(warnings),
|
||||
};
|
||||
}
|
||||
|
||||
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
if (kind === "sabnzbd") {
|
||||
const q = obj(obj(await sab("queue", "&limit=1")).queue);
|
||||
if (!("slots" in q) && !("status" in q)) throw new Error("That does not look like SABnzbd — is the address its own?");
|
||||
} else {
|
||||
await withQbSession(async (get) => {
|
||||
const v = (await get("/app/version")).text.trim();
|
||||
if (!/^v?\d+(\.\d+)+/.test(v)) throw new Error("That does not look like qBittorrent — is the address its own?");
|
||||
});
|
||||
}
|
||||
return { ok: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
return withDiagLogging(kind, { ping, read: kind === "sabnzbd" ? readSab : readQb });
|
||||
}
|
||||
@@ -141,6 +141,17 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
|
||||
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "Settings → General → Security → API Key" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
qbittorrent: [
|
||||
{ key: "url", label: "qBittorrent Web UI URL", secret: false, placeholder: "http://qbittorrent.example.lan:8080" },
|
||||
{ key: "username", label: "User name (optional)", secret: false, optional: true, placeholder: "empty when qBittorrent lets this address in without signing in" },
|
||||
{ key: "password", label: "Password (optional)", secret: true, optional: true, type: "password", placeholder: "Tools → Options → Web UI" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
sabnzbd: [
|
||||
{ key: "url", label: "SABnzbd URL", secret: false, placeholder: "http://sabnzbd.example.lan:8080" },
|
||||
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "Config → General → Security → API Key (the API key, not the NZB key)" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
adguard: [
|
||||
{ key: "url", label: "AdGuard Home URL", secret: false, placeholder: "http://adguard.example.lan:3000" },
|
||||
{ key: "username", label: "User name (optional)", secret: false, optional: true, placeholder: "empty when AdGuard Home has no sign-in" },
|
||||
|
||||
@@ -15,7 +15,18 @@ export interface JsonResponse {
|
||||
|
||||
const TIMEOUT_MS = 15_000;
|
||||
|
||||
export function requestJson(url: string, opts: { method?: "GET" | "POST" | "PUT" | "PATCH" | "DELETE"; headers?: Record<string, string>; body?: unknown; insecure?: boolean; timeoutMs?: number } = {}): Promise<JsonResponse> {
|
||||
export function requestJson(
|
||||
url: string,
|
||||
opts: {
|
||||
method?: "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
|
||||
headers?: Record<string, string>;
|
||||
body?: unknown;
|
||||
/** A body sent as it is (a form, say), with the Content-Type the caller names in `headers`; used instead of `body`. */
|
||||
rawBody?: string;
|
||||
insecure?: boolean;
|
||||
timeoutMs?: number;
|
||||
} = {},
|
||||
): Promise<JsonResponse> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let parsed: URL;
|
||||
try {
|
||||
@@ -26,7 +37,7 @@ export function requestJson(url: string, opts: { method?: "GET" | "POST" | "PUT"
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return reject(new Error("The address must start with http:// or https://"));
|
||||
const isHttps = parsed.protocol === "https:";
|
||||
const lib = isHttps ? https : http;
|
||||
const payload = opts.body === undefined ? null : JSON.stringify(opts.body);
|
||||
const payload = opts.rawBody !== undefined ? opts.rawBody : opts.body === undefined ? null : JSON.stringify(opts.body);
|
||||
const limit = opts.timeoutMs ?? TIMEOUT_MS;
|
||||
const started = Date.now();
|
||||
const method = opts.method ?? "GET";
|
||||
|
||||
@@ -26,6 +26,7 @@ import { createSpeedtestAdapter } from "./speedtest/adapter.js";
|
||||
import { createFragnetAdapter } from "./fragnet/adapter.js";
|
||||
import { createArrAdapter } from "./arr/adapter.js";
|
||||
import { createAdguardAdapter } from "./adguard/adapter.js";
|
||||
import { createDownloadAdapter } from "./downloads/adapter.js";
|
||||
import { createTechnitiumAdapter } from "./technitium/adapter.js";
|
||||
import { createGrafanaAdapter } from "./grafana/adapter.js";
|
||||
import { createNextcloudAdapter } from "./nextcloud/adapter.js";
|
||||
@@ -95,6 +96,9 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
|
||||
case "lidarr":
|
||||
case "readarr":
|
||||
return createArrAdapter(type, config as any);
|
||||
case "qbittorrent":
|
||||
case "sabnzbd":
|
||||
return createDownloadAdapter(type, config as any);
|
||||
case "adguard":
|
||||
return createAdguardAdapter(config as any);
|
||||
case "technitium":
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { Router } from "express";
|
||||
import { inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { downloadState, integrations } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { DOWNLOAD_KINDS, type DownloadKind } from "../integrations/downloads/adapter.js";
|
||||
import { downloadProblems, downloadSyncStatuses, isDownloadFresh, recentFailed, storedOf, stuckItems } from "../services/downloadsSync.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const downloadsRouter = Router();
|
||||
downloadsRouter.use(requireAuth);
|
||||
|
||||
/** Every qBittorrent and SABnzbd, how it is, from what the last reads stored: speed, what is in its queue and in what state, what is stuck or failed lately, its disk and what is wrong. Everyone signed in can see it. Nothing here changes anything in them. */
|
||||
downloadsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, states, statuses] = await Promise.all([
|
||||
db.select({ id: integrations.id, type: integrations.type, name: integrations.name, enabled: integrations.enabled, baseUrl: integrations.baseUrl }).from(integrations).where(inArray(integrations.type, DOWNLOAD_KINDS)).orderBy(integrations.name),
|
||||
db.select().from(downloadState),
|
||||
downloadSyncStatuses(),
|
||||
]);
|
||||
const byId = new Map(states.map((s) => [s.integrationId, s]));
|
||||
const now = Date.now();
|
||||
res.json({
|
||||
clients: rows.map((r) => {
|
||||
const row = byId.get(r.id);
|
||||
const state = row ? storedOf(row) : null;
|
||||
const status = statuses[r.id] ?? null;
|
||||
const fresh = isDownloadFresh(status ?? undefined, now);
|
||||
return {
|
||||
id: r.id,
|
||||
kind: r.type as DownloadKind,
|
||||
name: r.name,
|
||||
enabled: r.enabled,
|
||||
url: r.baseUrl,
|
||||
status,
|
||||
fresh,
|
||||
state: state && {
|
||||
...state,
|
||||
/** The ids of the items that are stuck, so the page marks them the way the alert counts them. */
|
||||
stuck: stuckItems(state.items, now).map((i) => i.id),
|
||||
failedRecent: recentFailed(state.failed, now).map((i) => i.id),
|
||||
},
|
||||
problems: state && fresh ? downloadProblems(r.type as DownloadKind, r.id, r.name, state, now) : [],
|
||||
};
|
||||
}),
|
||||
});
|
||||
}));
|
||||
@@ -43,6 +43,8 @@ import { forgetFragnet, syncFragnet } from "../services/fragnetSync.js";
|
||||
import { forgetArr, syncArr } from "../services/arrSync.js";
|
||||
import { ARR_KINDS } from "../integrations/arr/adapter.js";
|
||||
import { forgetAdguard, syncAdguard } from "../services/adguardSync.js";
|
||||
import { forgetDownload, syncDownload } from "../services/downloadsSync.js";
|
||||
import { DOWNLOAD_KINDS } from "../integrations/downloads/adapter.js";
|
||||
import { forgetTechnitium, syncTechnitium } from "../services/technitiumSync.js";
|
||||
import { forgetGrafana, syncGrafana } from "../services/grafanaSync.js";
|
||||
import { forgetNextcloud, syncNextcloud } from "../services/nextcloudSync.js";
|
||||
@@ -160,6 +162,7 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
|
||||
if (type === "speedtest") void syncSpeedtest(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "fragnet") void syncFragnet(created.id).then(() => invalidateAlertsCache());
|
||||
if ((ARR_KINDS as string[]).includes(type)) void syncArr(created.id).then(() => invalidateAlertsCache());
|
||||
if ((DOWNLOAD_KINDS as string[]).includes(type)) void syncDownload(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "adguard") void syncAdguard(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "technitium") void syncTechnitium(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "grafana") void syncGrafana(created.id).then(() => invalidateAlertsCache());
|
||||
@@ -279,6 +282,7 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
if (updated.type === "speedtest" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncSpeedtest(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "fragnet" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncFragnet(updated.id).then(() => invalidateAlertsCache());
|
||||
if ((ARR_KINDS as string[]).includes(updated.type) && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncArr(updated.id).then(() => invalidateAlertsCache());
|
||||
if ((DOWNLOAD_KINDS as string[]).includes(updated.type) && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncDownload(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "adguard" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncAdguard(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "technitium" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncTechnitium(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "grafana" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncGrafana(updated.id).then(() => invalidateAlertsCache());
|
||||
@@ -405,6 +409,7 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
if (existing.type === "speedtest") await forgetSpeedtest(id);
|
||||
if (existing.type === "fragnet") await forgetFragnet(id);
|
||||
if ((ARR_KINDS as string[]).includes(existing.type)) await forgetArr(id);
|
||||
if ((DOWNLOAD_KINDS as string[]).includes(existing.type)) await forgetDownload(id);
|
||||
if (existing.type === "adguard") await forgetAdguard(id);
|
||||
if (existing.type === "technitium") await forgetTechnitium(id);
|
||||
if (existing.type === "grafana") await forgetGrafana(id);
|
||||
|
||||
@@ -91,6 +91,8 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
lidarr: { label: "Lidarr", link: "/media" },
|
||||
readarr: { label: "Readarr", link: "/media" },
|
||||
prowlarr: { label: "Prowlarr", link: "/media" },
|
||||
qbittorrent: { label: "qBittorrent", link: "/downloads" },
|
||||
sabnzbd: { label: "SABnzbd", link: "/downloads" },
|
||||
adguard: { label: "AdGuard Home", link: "/adguard" },
|
||||
technitium: { label: "Technitium DNS", link: "/technitium" },
|
||||
versions: { label: "Versions", link: "/versions" },
|
||||
@@ -147,6 +149,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "grafana", id: Number(m[2]), category: m[1] === "ds" ? "integration" : "monitoring" };
|
||||
m = /^(sonarr|radarr|prowlarr|lidarr|readarr)-(health|queue|disk):(\d+)/.exec(key);
|
||||
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
|
||||
m = /^(qbittorrent|sabnzbd)-(stuck|connection|paused|stalled|disk):(\d+)/.exec(key);
|
||||
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
|
||||
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
|
||||
if (m) return { type: "fragnet", id: Number(m[2]), category: m[1] === "disk" ? "disk" : m[1] === "backup" ? "backup" : "monitoring" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
@@ -301,7 +305,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr|lidarr|readarr|adguard|technitium|grafana):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr|lidarr|readarr|qbittorrent|sabnzbd|adguard|technitium|grafana):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] === "images" ? " (age of the images)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
/**
|
||||
* Keeps what qBittorrent and SABnzbd say up to date: every five minutes each download client is read — how fast it is going, what is in its queue and in what state, whether it can reach the
|
||||
* network (qBittorrent) or is paused (SABnzbd), what failed lately (SABnzbd), the room in its download folder — and the state is stored. Pages and alerts are built from what is stored, so none of
|
||||
* them talks to the clients.
|
||||
*
|
||||
* What is raised: **downloads that are stuck** — in an error state (qBittorrent: an error, files gone missing), or stalled with nothing moving for a day (one alert for the lot, naming the first few);
|
||||
* **no connection** — qBittorrent has torrents but has reported no network at all for fifteen minutes (a VPN that dropped); **paused and forgotten** — SABnzbd's queue has been paused for six hours
|
||||
* with downloads waiting; **downloading nothing** — SABnzbd has work waiting and is not paused, yet nothing has come in for an hour (a news server that does not answer or does not accept the
|
||||
* login); **a disk nearly full** (under 10% free, critical under 5%; qBittorrent does not say how big the disk is, so there it is under 20 GB, critical under 5 GB). "Since when" is what this
|
||||
* app saw: the time of the first read that showed it.
|
||||
*/
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { downloadState, integrations } from "../db/schema.js";
|
||||
import { DOWNLOAD_KINDS, DOWNLOAD_NAMES, createDownloadAdapter, emptyCounts, type Connection, type DownloadDisk, type DownloadItem, type DownloadKind, type DownloadRead, type DownloadState } from "../integrations/downloads/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
const STATUS_FLAG = "downloadsSync";
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
/** A stalled torrent that has had no activity for this long is stuck, not just slow to find a source. */
|
||||
export const STALLED_AFTER_MS = 24 * HOUR;
|
||||
/** No network at all for this long is a dropped connection, not a restart. */
|
||||
export const DISCONNECTED_AFTER_MS = 15 * MIN;
|
||||
/** A queue paused for this long, with downloads waiting, is forgotten. */
|
||||
export const PAUSED_AFTER_MS = 6 * HOUR;
|
||||
/** Work waiting and nothing coming in for this long is a client that cannot get anything. */
|
||||
export const IDLE_AFTER_MS = HOUR;
|
||||
/** A failure is news for this long. */
|
||||
export const FAILED_RECENT_MS = 24 * HOUR;
|
||||
export const DISK_WARN_FREE = 0.1;
|
||||
export const DISK_CRITICAL_FREE = 0.05;
|
||||
/** Where the disk's size is not said: free space under this is a warning, under the second figure critical. */
|
||||
export const DISK_WARN_BYTES = 20 * 1024 ** 3;
|
||||
export const DISK_CRITICAL_BYTES = 5 * 1024 ** 3;
|
||||
/** A disk smaller than this is not a download disk (a tmpfs, a boot partition). */
|
||||
export const MIN_DISK_BYTES = 2 * 1024 ** 3;
|
||||
|
||||
export interface DownloadSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, DownloadSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, DownloadSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function downloadSyncStatuses(): Promise<Record<number, DownloadSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isDownloadFresh = (status: DownloadSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof downloadState.$inferSelect;
|
||||
|
||||
function parse<T>(stored: string | null): T | null {
|
||||
if (stored === null) return null;
|
||||
try {
|
||||
return JSON.parse(stored) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** What the app has seen between reads: the time of the first read that showed each of these; null while it is not so. */
|
||||
export interface DownloadSince {
|
||||
pausedSince: string | null;
|
||||
disconnectedSince: string | null;
|
||||
idleSince: string | null;
|
||||
}
|
||||
|
||||
/** What is stored of one download client, as the read gave it, plus what the app has seen between reads. */
|
||||
export interface StoredDownload {
|
||||
version: string | null;
|
||||
speedDown: number | null;
|
||||
speedUp: number | null;
|
||||
paused: boolean;
|
||||
connection: Connection | null;
|
||||
counts: Record<DownloadState, number>;
|
||||
total: number;
|
||||
items: DownloadItem[];
|
||||
failed: DownloadItem[] | null;
|
||||
disks: DownloadDisk[] | null;
|
||||
warnings: { count: number; recent: string[] } | null;
|
||||
since: DownloadSince;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
const NO_SINCE: DownloadSince = { pausedSince: null, disconnectedSince: null, idleSince: null };
|
||||
|
||||
export const storedOf = (row: StateRow): StoredDownload => ({
|
||||
version: row.version,
|
||||
speedDown: row.speedDown,
|
||||
speedUp: row.speedUp,
|
||||
paused: row.paused,
|
||||
connection: row.connection === "connected" || row.connection === "firewalled" || row.connection === "disconnected" ? row.connection : null,
|
||||
counts: { ...emptyCounts(), ...(parse<Record<DownloadState, number>>(row.counts) ?? {}) },
|
||||
total: row.total,
|
||||
items: parse<DownloadItem[]>(row.items) ?? [],
|
||||
failed: parse<DownloadItem[]>(row.failed),
|
||||
disks: parse<DownloadDisk[]>(row.disks),
|
||||
warnings: parse<{ count: number; recent: string[] }>(row.warnings),
|
||||
since: { ...NO_SINCE, ...(parse<Partial<DownloadSince>>(row.since) ?? {}) },
|
||||
updatedAt: row.updatedAt,
|
||||
});
|
||||
|
||||
/** Pure: items that have not finished and are not sitting idle — what is still to be done. */
|
||||
export const pendingCount = (counts: Record<DownloadState, number>, total: number): number => total - counts.seeding - counts.completed - counts.failed;
|
||||
|
||||
/** Pure: work is waiting for a connection to the news server, yet nothing is coming in. */
|
||||
export const isIdleWithWork = (r: Pick<DownloadRead, "paused" | "speedDown" | "counts">): boolean => !r.paused && r.speedDown === 0 && r.counts.downloading + r.counts.queued > 0;
|
||||
|
||||
/** Pure: what the app has seen between reads, carried from the last read. A thing first seen wrong starts its clock now. */
|
||||
export function carrySince(read: Pick<DownloadRead, "paused" | "connection" | "speedDown" | "counts" | "total">, previous: DownloadSince | null, now: Date): DownloadSince {
|
||||
const at = now.toISOString();
|
||||
return {
|
||||
pausedSince: read.paused ? (previous?.pausedSince ?? at) : null,
|
||||
disconnectedSince: read.connection === "disconnected" && read.total > 0 ? (previous?.disconnectedSince ?? at) : null,
|
||||
idleSince: isIdleWithWork(read) ? (previous?.idleSince ?? at) : null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
|
||||
export async function storeRead(integrationId: number, r: DownloadRead, now: Date = new Date()): Promise<void> {
|
||||
const [previous] = await db.select().from(downloadState).where(eq(downloadState.integrationId, integrationId)).limit(1);
|
||||
const since = carrySince(r, previous ? storedOf(previous).since : null, now);
|
||||
const values = {
|
||||
version: r.version,
|
||||
speedDown: r.speedDown === null ? null : Math.round(r.speedDown),
|
||||
speedUp: r.speedUp === null ? null : Math.round(r.speedUp),
|
||||
paused: r.paused,
|
||||
connection: r.connection,
|
||||
counts: JSON.stringify(r.counts),
|
||||
total: r.total,
|
||||
items: JSON.stringify(r.items),
|
||||
failed: r.failed && JSON.stringify(r.failed),
|
||||
disks: r.disks && JSON.stringify(r.disks),
|
||||
warnings: r.warnings && JSON.stringify(r.warnings),
|
||||
since: JSON.stringify(since),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(downloadState).values({ integrationId, ...values }).onConflictDoUpdate({ target: downloadState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one download client and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncDownload(integrationId: number, now: Date = new Date(), read?: (kind: DownloadKind, config: Record<string, unknown>) => Promise<DownloadRead>): Promise<DownloadSyncStatus> {
|
||||
let status: DownloadSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
const kind = loaded?.integration.type as DownloadKind | undefined;
|
||||
if (!loaded || !kind || !DOWNLOAD_KINDS.includes(kind)) throw new Error("not a qBittorrent or SABnzbd integration");
|
||||
const result = await (read ? read(kind, loaded.config) : createDownloadAdapter(kind, loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[downloads] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[downloads] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllDownloads(now: Date = new Date()): Promise<Record<number, DownloadSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(inArray(integrations.type, DOWNLOAD_KINDS), eq(integrations.enabled, true)));
|
||||
const out: Record<number, DownloadSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncDownload(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetDownload(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface DownloadProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const slug = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40);
|
||||
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
|
||||
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / (24 * HOUR)), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
|
||||
|
||||
export const sizeText = (bytes: number): string => {
|
||||
const gb = bytes / 1024 ** 3;
|
||||
return gb >= 1024 ? `${(gb / 1024).toFixed(1)} TB` : `${gb.toFixed(gb >= 100 ? 0 : 1)} GB`;
|
||||
};
|
||||
|
||||
/** Pure: how long ago anything moved in a stalled item — its last activity, or when it was added; null when neither is known. */
|
||||
function idleMs(i: DownloadItem, nowMs: number): number | null {
|
||||
const at = i.activityAt ?? i.addedAt;
|
||||
return at === null ? null : nowMs - Date.parse(at);
|
||||
}
|
||||
|
||||
/** Pure: whether an item is stuck — in an error state, or stalled with nothing moving for a day. */
|
||||
export function isStuck(i: DownloadItem, nowMs: number): boolean {
|
||||
if (i.state === "error") return true;
|
||||
if (i.state !== "stalled") return false;
|
||||
const idle = idleMs(i, nowMs);
|
||||
return idle !== null && idle >= STALLED_AFTER_MS;
|
||||
}
|
||||
|
||||
/** Pure: the stuck items, the ones in error first, then the longest idle. */
|
||||
export const stuckItems = (items: DownloadItem[], nowMs: number): DownloadItem[] =>
|
||||
items.filter((i) => isStuck(i, nowMs)).sort((a, b) => Number(b.state === "error") - Number(a.state === "error") || (idleMs(b, nowMs) ?? 0) - (idleMs(a, nowMs) ?? 0));
|
||||
|
||||
/** Pure: the downloads that failed in the last day. */
|
||||
export const recentFailed = (failed: DownloadItem[] | null, nowMs: number): DownloadItem[] => (failed ?? []).filter((f) => f.activityAt !== null && nowMs - Date.parse(f.activityAt) <= FAILED_RECENT_MS);
|
||||
|
||||
/** Pure: what is wrong with one download client, from what is stored for it. */
|
||||
export function downloadProblems(kind: DownloadKind, id: number, name: string, s: StoredDownload, nowMs: number): DownloadProblem[] {
|
||||
const out: DownloadProblem[] = [];
|
||||
const app = DOWNLOAD_NAMES[kind];
|
||||
|
||||
const stuck = stuckItems(s.items, nowMs);
|
||||
if (stuck.length > 0) {
|
||||
const names = stuck.slice(0, 3).map((i) => i.name);
|
||||
const worst = stuck[0];
|
||||
const why = worst.state === "error" ? (worst.rawState === "missingFiles" ? "its files are missing" : "it is in an error state") : `nothing has moved for ${ago(idleMs(worst, nowMs) ?? 0)}`;
|
||||
out.push({ key: `${kind}-stuck:${id}`, severity: "warning", message: `${name}: ${plural(stuck.length, "download")} ${stuck.length === 1 ? "is" : "are"} stuck — ${names.join("; ")}${stuck.length > 3 ? `; and ${stuck.length - 3} more` : ""} (${why})` });
|
||||
}
|
||||
|
||||
if (s.since.disconnectedSince !== null && s.connection === "disconnected" && nowMs - Date.parse(s.since.disconnectedSince) >= DISCONNECTED_AFTER_MS) {
|
||||
out.push({ key: `${kind}-connection:${id}`, severity: "warning", message: `${name}: ${app} has had no connection to the network for ${ago(nowMs - Date.parse(s.since.disconnectedSince))} — if it is bound to a VPN, is the VPN up?` });
|
||||
}
|
||||
|
||||
const pending = pendingCount(s.counts, s.total);
|
||||
if (s.paused && pending > 0 && s.since.pausedSince !== null && nowMs - Date.parse(s.since.pausedSince) >= PAUSED_AFTER_MS) {
|
||||
out.push({ key: `${kind}-paused:${id}`, severity: "warning", message: `${name}: ${app} has been paused for ${ago(nowMs - Date.parse(s.since.pausedSince))} with ${plural(pending, "download")} waiting` });
|
||||
}
|
||||
|
||||
if (!s.paused && s.speedDown === 0 && s.since.idleSince !== null && nowMs - Date.parse(s.since.idleSince) >= IDLE_AFTER_MS) {
|
||||
const waiting = s.counts.downloading + s.counts.queued;
|
||||
out.push({
|
||||
key: `${kind}-stalled:${id}`,
|
||||
severity: "warning",
|
||||
message: `${name}: ${app} has downloaded nothing for ${ago(nowMs - Date.parse(s.since.idleSince))} although ${plural(waiting, "download")} ${waiting === 1 ? "is" : "are"} waiting — can it reach its news server?`,
|
||||
});
|
||||
}
|
||||
|
||||
for (const d of s.disks ?? []) {
|
||||
if (d.total !== null) {
|
||||
if (d.total < MIN_DISK_BYTES) continue;
|
||||
const free = d.free / d.total;
|
||||
if (free < DISK_WARN_FREE) {
|
||||
out.push({ key: `${kind}-disk:${id}:${slug(d.label)}`, severity: free < DISK_CRITICAL_FREE ? "critical" : "warning", message: `${name}: the ${d.label.toLowerCase()} of ${app} has ${sizeText(d.free)} free of ${sizeText(d.total)} (${Math.floor(free * 100)}%)` });
|
||||
}
|
||||
} else if (d.free < DISK_WARN_BYTES) {
|
||||
out.push({ key: `${kind}-disk:${id}:${slug(d.label)}`, severity: d.free < DISK_CRITICAL_BYTES ? "critical" : "warning", message: `${name}: the ${d.label.toLowerCase()} of ${app} has only ${sizeText(d.free)} free` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initDownloadsSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllDownloads().catch((err) => console.error("[downloads] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 43_000);
|
||||
console.log(`[downloads] reading qBittorrent and SABnzbd every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { technitiumState, authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState , adguardState , grafanaState } from "../db/schema.js";
|
||||
import { downloadState, technitiumState, authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState , adguardState , grafanaState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -39,6 +39,8 @@ import { adguardProblems, adguardSyncStatuses, isAdguardFresh, storedOf as adgua
|
||||
import { isTechnitiumFresh, storedOf as technitiumStoredOf, technitiumProblems, technitiumSyncStatuses, type TechnitiumProblem } from "./technitiumSync.js";
|
||||
import { grafanaProblems, grafanaSyncStatuses, isGrafanaFresh, storedOf as grafanaStoredOf, type GrafanaProblem } from "./grafanaSync.js";
|
||||
import { arrProblems, arrSyncStatuses, isArrFresh, storedOf as arrStoredOf, type ArrProblem } from "./arrSync.js";
|
||||
import { DOWNLOAD_KINDS, type DownloadKind } from "../integrations/downloads/adapter.js";
|
||||
import { downloadProblems, downloadSyncStatuses, isDownloadFresh, storedOf as downloadStoredOf, type DownloadProblem } from "./downloadsSync.js";
|
||||
import { fragnetProblems, fragnetSyncStatuses, isFragnetFresh, optionsOf as fragnetOptions, serversOf as fragnetServersOf, type FragnetProblem } from "./fragnetSync.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
@@ -140,6 +142,8 @@ export interface HealthSnapshot {
|
||||
fragnet?: { integrationId: number; problems: FragnetProblem[] }[];
|
||||
/** What is wrong with each Sonarr, Radarr and Prowlarr (what it says itself, downloads that are stuck, a disk nearly full), only while that reading is recent (services/arrSync.ts). */
|
||||
arr?: { integrationId: number; kind: ArrKind; problems: ArrProblem[] }[];
|
||||
/** What is wrong with each qBittorrent and SABnzbd (downloads that are stuck, no connection, a queue left paused or getting nothing, a disk nearly full), only while that reading is recent (services/downloadsSync.ts). */
|
||||
downloads?: { integrationId: number; kind: DownloadKind; problems: DownloadProblem[] }[];
|
||||
/** What is wrong with each AdGuard Home (DNS not running or not answering, protection left off, filter lists not updating), only while that reading is recent (services/adguardSync.ts). */
|
||||
adguard?: { integrationId: number; problems: AdguardProblem[] }[];
|
||||
/** What is wrong with each Technitium DNS Server (DNS not answering, blocking left off, block lists not updating, zones expired or not syncing), only while that reading is recent (services/technitiumSync.ts). */
|
||||
@@ -297,6 +301,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
for (const t of snapshot.technitium ?? []) for (const p of t.problems) out.push({ key: p.key, source: `technitium:${t.integrationId}`, message: p.message, severity: p.severity });
|
||||
for (const g of snapshot.grafana ?? []) for (const p of g.problems) out.push({ key: p.key, source: `grafana:${g.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// qBittorrent and SABnzbd: downloads that are stuck, no connection, a queue left paused or getting nothing, disks nearly full.
|
||||
for (const d of snapshot.downloads ?? []) for (const p of d.problems) out.push({ key: p.key, source: `${d.kind}:${d.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// Sonarr, Radarr and Prowlarr: what they say is wrong, downloads that are stuck, disks nearly full.
|
||||
for (const a of snapshot.arr ?? []) for (const p of a.problems) out.push({ key: p.key, source: `${a.kind}:${a.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
@@ -686,6 +693,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
console.error("[health] couldn't judge Sonarr, Radarr and Prowlarr:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// qBittorrent and SABnzbd: read by their own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsD = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(and(inArray(integrations.type, DOWNLOAD_KINDS), eq(integrations.enabled, true)));
|
||||
if (rowsD.length > 0) {
|
||||
const statuses = await downloadSyncStatuses();
|
||||
for (const r of rowsD) {
|
||||
const [state] = await db.select().from(downloadState).where(eq(downloadState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isDownloadFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`${r.type}:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.downloads ??= []).push({ integrationId: r.id, kind: r.type as DownloadKind, problems: downloadProblems(r.type as DownloadKind, r.id, r.name, downloadStoredOf(state), Date.now()) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge qBittorrent and SABnzbd:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Fragnet game servers: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsF = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "fragnet"), eq(integrations.enabled, true)));
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
*/
|
||||
export const INTEGRATION_GROUPS: { id: string; label: string; types: string[] }[] = [
|
||||
{ id: "infrastructure", label: "Infrastructure", types: ["proxmox", "synology", "dockhand", "hetzner", "fragnet", "tailscale"] },
|
||||
{ id: "apps", label: "Apps", types: ["homeassistant", "jellyfin", "sonarr", "radarr", "lidarr", "readarr", "prowlarr", "immich", "paperless", "nextcloud", "authentik"] },
|
||||
{ id: "apps", label: "Apps", types: ["homeassistant", "jellyfin", "sonarr", "radarr", "lidarr", "readarr", "prowlarr", "qbittorrent", "sabnzbd", "immich", "paperless", "nextcloud", "authentik"] },
|
||||
{ id: "backups", label: "Backups", types: ["pbs"] },
|
||||
{ id: "network", label: "Network", types: ["unifi", "npm", "phpipam", "netbox", "adguard", "technitium"] },
|
||||
{ id: "monitoring", label: "Monitoring", types: ["uptimekuma", "prometheus", "grafana", "ups", "speedtest"] },
|
||||
|
||||
@@ -45,6 +45,8 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^(?:sonarr|radarr|prowlarr|lidarr|readarr)-(?:health|queue|disk):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^(?:qbittorrent|sabnzbd)-(?:stuck|connection|paused|stalled|disk):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^version-behind:(\d+)$/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^adguard-(?:stopped|dns|protection|filters):(\d+)/.exec(key);
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
import { eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { technitiumState, assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState, adguardState, grafanaState } from "../db/schema.js";
|
||||
import { downloadState, technitiumState, assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState, adguardState, grafanaState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
@@ -15,6 +15,8 @@ import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
|
||||
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
|
||||
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf, loadBalancersOf as hetznerLoadBalancersOf, certificatesOf as hetznerCertificatesOf, trafficForecast as hetznerTrafficForecast } from "./hetznerSync.js";
|
||||
import { ARR_KINDS } from "../integrations/arr/adapter.js";
|
||||
import { DOWNLOAD_KINDS, DOWNLOAD_STATES } from "../integrations/downloads/adapter.js";
|
||||
import { downloadSyncStatuses, isDownloadFresh, recentFailed as downloadsFailed, storedOf as downloadStoredOf, stuckItems as downloadStuck } from "./downloadsSync.js";
|
||||
import { versionRows as appVersionRows } from "./appVersions.js";
|
||||
import { adguardSyncStatuses, isAdguardFresh, storedOf as adguardStoredOf } from "./adguardSync.js";
|
||||
import { blockListsOverdueMs, isTechnitiumFresh, storedOf as technitiumStoredOf, technitiumSyncStatuses } from "./technitiumSync.js";
|
||||
@@ -210,6 +212,26 @@ async function addArrMetrics(out: Exposition, now: Date) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each qBittorrent and SABnzbd, for /metrics: how fast it is going, the items in each state and how many are stuck, whether it is connected or paused, what failed in the last day and the free space of its disk. Only while the reading is recent. */
|
||||
async function addDownloadMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await downloadSyncStatuses();
|
||||
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations).where(inArray(integrations.type, DOWNLOAD_KINDS))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(downloadState)) {
|
||||
const row = rows.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isDownloadFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const state = downloadStoredOf(s);
|
||||
const labels = { client: row.type, name: row.name };
|
||||
if (state.speedDown !== null) out.add("homelab_download_speed_bytes", "gauge", "How fast the download client is downloading, in bytes per second.", state.speedDown, { ...labels, direction: "down" });
|
||||
if (state.speedUp !== null) out.add("homelab_download_speed_bytes", "gauge", "How fast the download client is downloading, in bytes per second.", state.speedUp, { ...labels, direction: "up" });
|
||||
for (const st of DOWNLOAD_STATES) if (state.counts[st] > 0 || st === "downloading" || st === "queued") out.add("homelab_download_items", "gauge", "How many items the download client has in each state.", state.counts[st], { ...labels, state: st });
|
||||
out.add("homelab_download_stuck_items", "gauge", "How many items are stuck (in an error state, or stalled with nothing moving for a day).", downloadStuck(state.items, now.getTime()).length, labels);
|
||||
out.add("homelab_download_paused", "gauge", "1 if the download client's whole queue is paused, else 0.", state.paused ? 1 : 0, labels);
|
||||
if (state.connection !== null) out.add("homelab_download_connected", "gauge", "1 if the download client can reach the network (not 'disconnected'), else 0.", state.connection === "disconnected" ? 0 : 1, labels);
|
||||
if (state.failed !== null) out.add("homelab_download_failed_recent", "gauge", "How many downloads failed in the last day.", downloadsFailed(state.failed, now.getTime()).length, labels);
|
||||
for (const d of state.disks ?? []) out.add("homelab_download_disk_free_bytes", "gauge", "Free space of a folder the download client uses, in bytes.", d.free, { ...labels, folder: d.label });
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Fragnet game server, for /metrics: whether it is running, its disk and memory against their limits and when it was last backed up. Only while the reading is recent. */
|
||||
async function addFragnetMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await fragnetSyncStatuses();
|
||||
@@ -586,6 +608,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addSpeedtestMetrics(out, now);
|
||||
await addFragnetMetrics(out, now);
|
||||
await addArrMetrics(out, now);
|
||||
await addDownloadMetrics(out, now);
|
||||
await addAdguardMetrics(out, now);
|
||||
await addTechnitiumMetrics(out, now);
|
||||
await addVersionMetrics(out);
|
||||
|
||||
@@ -0,0 +1,584 @@
|
||||
/**
|
||||
* qBittorrent and SABnzbd: reading what their APIs say, what is stored, what is wrong (stuck downloads, no connection, a queue left paused or getting nothing, a disk nearly full), the health
|
||||
* pass, the alerts, the page's data and /metrics.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type IncomingMessage, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-downloads-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, integrations, downloadState } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const adapter = await import("../src/integrations/downloads/adapter.js");
|
||||
const dl = await import("../src/services/downloadsSync.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { downloadsRouter } = await import("../src/routes/downloads.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { INTEGRATION_FIELDS, validateIntegrationConfig, splitIntegrationConfig } = await import("../src/integrations/fieldSchemas.js");
|
||||
const { createIntegrationAdapter } = await import("../src/integrations/registry.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
const DAY = 24 * HOUR;
|
||||
const GB = 1024 ** 3;
|
||||
const iso = (ms: number) => new Date(ms).toISOString();
|
||||
|
||||
type Item = import("../src/integrations/downloads/adapter.js").DownloadItem;
|
||||
type Read = import("../src/integrations/downloads/adapter.js").DownloadRead;
|
||||
|
||||
const item = (extra: Partial<Item> = {}): Item => ({ id: "h1", name: "Some.Download", state: "downloading", rawState: "downloading", progress: 0.5, size: 1000, left: 500, speed: 100, eta: 60, addedAt: iso(Date.now() - HOUR), activityAt: iso(Date.now() - MIN), category: "tv", message: null, ...extra });
|
||||
/** A read built from its items, the way the adapter builds one. */
|
||||
const read = (items: Item[] = [item()], extra: Partial<Read> = {}): Read => ({
|
||||
version: "4.6.2",
|
||||
speedDown: 1000,
|
||||
speedUp: 10,
|
||||
paused: false,
|
||||
connection: "connected",
|
||||
...adapter.summarise(items),
|
||||
failed: null,
|
||||
disks: [{ label: "Download folder", free: 500 * GB, total: null }],
|
||||
warnings: null,
|
||||
...extra,
|
||||
});
|
||||
|
||||
describe("what the APIs say", () => {
|
||||
test("a torrent: its state in the shared words, progress, size, speed, 'never' as no ETA, the times, the category", () => {
|
||||
const t = adapter.parseTorrent({ hash: "abc", name: "Linux ISO", state: "stalledDL", progress: 0.25, size: 1000, amount_left: 750, dlspeed: 0, eta: 8640000, added_on: 1780000000, last_activity: 1780000500, category: "iso" })!;
|
||||
assert.deepEqual([t.id, t.name, t.state, t.rawState, t.progress, t.size, t.left, t.speed, t.eta, t.category], ["abc", "Linux ISO", "stalled", "stalledDL", 0.25, 1000, 750, 0, null, "iso"]);
|
||||
assert.deepEqual([t.addedAt, t.activityAt], [new Date(1780000000 * 1000).toISOString(), new Date(1780000500 * 1000).toISOString()]);
|
||||
assert.equal(adapter.parseTorrent({ hash: "a", state: "downloading", eta: 90 })!.eta, 90);
|
||||
assert.equal(adapter.parseTorrent({ hash: "a", state: "downloading", progress: 1.4 })!.progress, 1);
|
||||
assert.equal(adapter.parseTorrent({ hash: "a", last_activity: 0 })!.activityAt, null, "0 is never");
|
||||
assert.equal(adapter.parseTorrent({ name: "no hash" }), null);
|
||||
});
|
||||
|
||||
test("every state qBittorrent has is placed; one it does not know is 'other'", () => {
|
||||
const state = (s: string) => adapter.parseTorrent({ hash: "a", state: s })!.state;
|
||||
const expected: Record<string, string> = {
|
||||
error: "error", missingFiles: "error", downloading: "downloading", forcedDL: "downloading", metaDL: "downloading", stalledDL: "stalled", queuedDL: "queued", pausedDL: "paused", stoppedDL: "paused",
|
||||
checkingDL: "processing", checkingUP: "processing", checkingResumeData: "processing", allocating: "processing", moving: "processing", uploading: "seeding", forcedUP: "seeding", stalledUP: "seeding", queuedUP: "seeding", pausedUP: "completed", stoppedUP: "completed", somethingNew: "other",
|
||||
};
|
||||
for (const [raw, shared] of Object.entries(expected)) assert.equal(state(raw), shared, raw);
|
||||
});
|
||||
|
||||
test("maindata: its torrents are keyed by hash, and the numbers about itself are read the same from server_state or transfer/info", () => {
|
||||
const items = adapter.parseMaindataTorrents({ torrents: { h1: { name: "A", state: "downloading" }, h2: { name: "B", state: "uploading" } } });
|
||||
assert.deepEqual(items.map((i) => [i.id, i.name, i.state]), [["h1", "A", "downloading"], ["h2", "B", "seeding"]]);
|
||||
assert.deepEqual(adapter.parseMaindataTorrents({}), []);
|
||||
assert.deepEqual(adapter.parseTransfer({ dl_info_speed: 5, up_info_speed: 2, connection_status: "Firewalled", free_space_on_disk: 42 }), { speedDown: 5, speedUp: 2, connection: "firewalled", free: 42 });
|
||||
assert.deepEqual(adapter.parseTransfer({ connection_status: "nonsense", free_space_on_disk: -1 }), { speedDown: null, speedUp: null, connection: null, free: null });
|
||||
});
|
||||
|
||||
test("the summary counts everything, and lists only what is not finished and idle — the ones with something to say first", () => {
|
||||
const all = [item({ id: "1", state: "seeding", name: "s" }), item({ id: "2", state: "downloading", name: "d" }), item({ id: "3", state: "error", name: "e" }), item({ id: "4", state: "completed", name: "c" }), item({ id: "5", state: "stalled", name: "t" }), item({ id: "6", state: "queued", name: "q" })];
|
||||
const s = adapter.summarise(all);
|
||||
assert.equal(s.total, 6);
|
||||
assert.deepEqual([s.counts.seeding, s.counts.completed, s.counts.error, s.counts.downloading, s.counts.stalled, s.counts.queued], [1, 1, 1, 1, 1, 1]);
|
||||
assert.deepEqual(s.items.map((i) => i.name), ["e", "t", "d", "q"]);
|
||||
assert.equal(adapter.summarise(Array.from({ length: 150 }, (_, n) => item({ id: String(n), name: `n${n}` }))).items.length, adapter.MAX_ITEMS);
|
||||
});
|
||||
|
||||
test("a SABnzbd slot: its figures are text in megabytes; '*' is no category; 'propagating' waits; an estimate of 0 is none", () => {
|
||||
const s = adapter.parseSabSlot({ nzo_id: "SABnzbd_nzo_1", filename: "Show.S01E01", status: "Downloading", percentage: "40", mb: "1000.0", mbleft: "600.0", cat: "*", timeleft: "0:10:00" })!;
|
||||
assert.deepEqual([s.id, s.name, s.state, s.progress, s.size, s.left, s.category, s.eta], ["SABnzbd_nzo_1", "Show.S01E01", "downloading", 0.4, 1000 * 1024 ** 2, 600 * 1024 ** 2, null, 600]);
|
||||
assert.equal(adapter.parseSabSlot({ nzo_id: "x", status: "Propagating" })!.state, "waiting");
|
||||
assert.equal(adapter.parseSabSlot({ nzo_id: "x", status: "Verifying" })!.state, "processing");
|
||||
assert.equal(adapter.parseSabSlot({ nzo_id: "x", status: "Paused", cat: "tv" })!.category, "tv");
|
||||
assert.equal(adapter.parseSabSlot({ nzo_id: "x", mb: "100", mbleft: "25" })!.progress, 0.75, "without a percentage, from what is left");
|
||||
assert.equal(adapter.parseSabSlot({ filename: "no id" }), null);
|
||||
assert.deepEqual([adapter.parseTimeLeft("0:00:00"), adapter.parseTimeLeft("1:02:03"), adapter.parseTimeLeft("1:00:00:10"), adapter.parseTimeLeft("soon"), adapter.parseTimeLeft(null)], [null, 3723, 86410, null, null]);
|
||||
});
|
||||
|
||||
test("the SABnzbd queue: version, paused or not, speed in bytes, the slots, and the two folders' disks — the same disk once", () => {
|
||||
const q = adapter.parseSabQueue({ queue: { version: "4.3.2", paused: false, status: "Downloading", kbpersec: "1500.5", diskspace1: "100.0", diskspacetotal1: "1000.0", diskspace2: "250.0", diskspacetotal2: "500.0", slots: [{ nzo_id: "a", filename: "A", status: "Queued" }, { filename: "no id" }] } });
|
||||
assert.deepEqual([q.version, q.paused, q.speedDown, q.items.map((i) => i.id)], ["4.3.2", false, Math.round(1500.5 * 1024), ["a"]]);
|
||||
assert.deepEqual(q.disks, [{ label: "Incomplete folder", free: 100 * GB, total: 1000 * GB }, { label: "Complete folder", free: 250 * GB, total: 500 * GB }]);
|
||||
const same = adapter.parseSabQueue({ queue: { diskspace1: "100", diskspacetotal1: "1000", diskspace2: "100", diskspacetotal2: "1000", slots: [] } });
|
||||
assert.deepEqual(same.disks?.map((d) => d.label), ["Incomplete folder"]);
|
||||
assert.equal(adapter.parseSabQueue({ queue: { paused: true } }).paused, true);
|
||||
assert.equal(adapter.parseSabQueue({ queue: { status: "Paused" } }).paused, true);
|
||||
assert.deepEqual([adapter.parseSabQueue({}).items, adapter.parseSabQueue({}).disks, adapter.parseSabQueue({}).speedDown], [[], null, null]);
|
||||
});
|
||||
|
||||
test("the SABnzbd history: only what failed, the newest first, with why; its warnings are lines or objects", () => {
|
||||
const f = adapter.parseSabFailed({ history: { slots: [{ nzo_id: "a", name: "Old", status: "Failed", fail_message: "Aborted, cannot be completed", completed: 1780000000, bytes: 5, category: "*" }, { nzo_id: "b", name: "Fine", status: "Completed", completed: 1780000900 }, { nzo_id: "c", name: "New", status: "Failed", completed: 1780000800 }] } });
|
||||
assert.deepEqual(f.map((x) => [x.name, x.state, x.message, x.category]), [["New", "failed", null, null], ["Old", "failed", "Aborted, cannot be completed", null]]);
|
||||
const w = adapter.parseSabWarnings({ warnings: ["first", { text: "second line", type: "WARNING" }, 5, { text: "third" }] });
|
||||
assert.deepEqual([w.count, w.recent], [3, ["third", "second line", "first"]]);
|
||||
assert.deepEqual(adapter.parseSabWarnings(null), { count: 0, recent: [] });
|
||||
});
|
||||
|
||||
describe("qBittorrent, answering", () => {
|
||||
let fake: Server;
|
||||
let url = "";
|
||||
let password = "secret";
|
||||
let banned = false;
|
||||
let maindata = true;
|
||||
const seen: { method: string | undefined; path: string; cookie: string | undefined; referer: string | undefined; body: string }[] = [];
|
||||
const body = (req: IncomingMessage) =>
|
||||
new Promise<string>((resolve) => {
|
||||
let text = "";
|
||||
req.on("data", (c) => (text += c));
|
||||
req.on("end", () => resolve(text));
|
||||
});
|
||||
before(async () => {
|
||||
fake = createHttpServer(async (req, res) => {
|
||||
const b = await body(req);
|
||||
const u = new URL(req.url ?? "", "http://x");
|
||||
seen.push({ method: req.method, path: u.pathname + u.search, cookie: req.headers.cookie, referer: req.headers.referer as string | undefined, body: b });
|
||||
const text = (code: number, t: string, headers: Record<string, string> = {}) => {
|
||||
res.writeHead(code, { "content-type": "text/plain", ...headers });
|
||||
res.end(t);
|
||||
};
|
||||
const json = (code: number, v: unknown) => {
|
||||
res.writeHead(code, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify(v));
|
||||
};
|
||||
if (u.pathname === "/api/v2/auth/login") {
|
||||
if (banned) return text(403, "Your IP address has been banned after too many failed authentication attempts.");
|
||||
const form = new URLSearchParams(b);
|
||||
return form.get("username") === "admin" && form.get("password") === password ? text(200, "Ok.", { "set-cookie": "SID=abc123; HttpOnly; path=/" }) : text(200, "Fails.");
|
||||
}
|
||||
if (u.pathname === "/api/v2/auth/logout") return text(200, "");
|
||||
if (req.headers.cookie !== "SID=abc123") return text(403, "Forbidden");
|
||||
if (u.pathname === "/api/v2/app/version") return text(200, "v4.6.2");
|
||||
if (u.pathname === "/api/v2/sync/maindata")
|
||||
return maindata
|
||||
? json(200, {
|
||||
rid: 1,
|
||||
full_update: true,
|
||||
torrents: {
|
||||
h1: { name: "Linux ISO", state: "downloading", progress: 0.5, size: 2000, amount_left: 1000, dlspeed: 1000, eta: 600, added_on: 1780000000, last_activity: 1780000500 },
|
||||
h2: { name: "Stuck one", state: "stalledDL", progress: 0.1, size: 100, amount_left: 90, last_activity: 1700000000 },
|
||||
h3: { name: "Seeded", state: "uploading", progress: 1, size: 50 },
|
||||
h4: { name: "Broken", state: "missingFiles", progress: 0.3, size: 70 },
|
||||
},
|
||||
server_state: { free_space_on_disk: 50 * GB, dl_info_speed: 5000, up_info_speed: 200, connection_status: "connected" },
|
||||
})
|
||||
: text(404, "");
|
||||
if (u.pathname === "/api/v2/torrents/info") return json(200, [{ hash: "z1", name: "Fallback", state: "downloading", progress: 0.2, size: 10 }]);
|
||||
if (u.pathname === "/api/v2/transfer/info") return json(200, { dl_info_speed: 7, up_info_speed: 3, connection_status: "firewalled" });
|
||||
text(404, "");
|
||||
});
|
||||
await new Promise<void>((resolve) => fake.listen(0, "127.0.0.1", resolve));
|
||||
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => fake.close());
|
||||
|
||||
test("it signs in with the form, sends the cookie and its own address as the referer, reads everything, and signs out", async () => {
|
||||
seen.length = 0;
|
||||
const r = await adapter.createDownloadAdapter("qbittorrent", { url, username: "admin", password: "secret" }).read();
|
||||
assert.deepEqual([r.version, r.speedDown, r.speedUp, r.connection, r.total, r.paused], ["4.6.2", 5000, 200, "connected", 4, false]);
|
||||
assert.deepEqual([r.counts.downloading, r.counts.stalled, r.counts.seeding, r.counts.error], [1, 1, 1, 1]);
|
||||
assert.deepEqual(r.items.map((i) => i.name), ["Broken", "Stuck one", "Linux ISO"], "the seeding one is not listed");
|
||||
assert.deepEqual(r.disks, [{ label: "Download folder", free: 50 * GB, total: null }]);
|
||||
assert.equal(r.failed, null);
|
||||
const login = seen[0];
|
||||
assert.deepEqual([login.method, login.path, login.referer], ["POST", "/api/v2/auth/login", `${url}/`]);
|
||||
assert.equal(new URLSearchParams(login.body).get("password"), "secret");
|
||||
assert.ok(seen.slice(1, -1).every((s) => s.method === "GET" && s.cookie === "SID=abc123"), "everything after is a GET with the cookie");
|
||||
assert.equal(seen[seen.length - 1].path, "/api/v2/auth/logout");
|
||||
});
|
||||
|
||||
test("without a user name it does not sign in at all (an address it lets in), and a refusal then says what to do", async () => {
|
||||
seen.length = 0;
|
||||
await assert.rejects(() => adapter.createDownloadAdapter("qbittorrent", { url }).read(), /did not let this address in \(HTTP 403\).*user name and password/);
|
||||
assert.ok(!seen.some((s) => s.path.includes("auth/login")));
|
||||
});
|
||||
|
||||
test("when maindata is not answered, the torrents are asked for on their own, and the numbers come from transfer/info", async () => {
|
||||
maindata = false;
|
||||
const r = await adapter.createDownloadAdapter("qbittorrent", { url, username: "admin", password: "secret" }).read();
|
||||
assert.deepEqual([r.items.map((i) => i.name), r.speedDown, r.connection, r.disks], [["Fallback"], 7, "firewalled", null]);
|
||||
maindata = true;
|
||||
});
|
||||
|
||||
test("a wrong password, a ban, and an address that is not qBittorrent are said — and a wrong password is said before anything is read", async () => {
|
||||
seen.length = 0;
|
||||
const bad = adapter.createDownloadAdapter("qbittorrent", { url, username: "admin", password: "nope" });
|
||||
assert.match((await bad.ping()).error ?? "", /did not accept the user name and password/);
|
||||
assert.equal(seen.length, 1);
|
||||
banned = true;
|
||||
assert.match((await bad.ping()).error ?? "", /refused the sign-in \(HTTP 403\).*bans an address/);
|
||||
banned = false;
|
||||
assert.equal((await adapter.createDownloadAdapter("qbittorrent", { url, username: "admin", password: "secret" }).ping()).ok, true);
|
||||
assert.equal((await adapter.createDownloadAdapter("qbittorrent", { url: "http://127.0.0.1:1", username: "a", password: "b" }).ping()).ok, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SABnzbd, answering", () => {
|
||||
let fake: Server;
|
||||
let url = "";
|
||||
let historyStatus = 200;
|
||||
const seen: { path: string; key: string | null; mode: string | null }[] = [];
|
||||
before(async () => {
|
||||
fake = createHttpServer((req, res) => {
|
||||
const u = new URL(req.url ?? "", "http://x");
|
||||
seen.push({ path: u.pathname, key: u.searchParams.get("apikey"), mode: u.searchParams.get("mode") });
|
||||
const send = (code: number, v: unknown) => {
|
||||
res.writeHead(code, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify(v));
|
||||
};
|
||||
if (u.pathname !== "/api") return send(404, {});
|
||||
if (u.searchParams.get("apikey") !== "good") return send(200, { status: false, error: "API Key Incorrect" });
|
||||
const mode = u.searchParams.get("mode");
|
||||
if (mode === "queue") return send(200, { queue: { version: "4.3.2", paused: true, status: "Paused", kbpersec: "0.00", diskspace1: "80.0", diskspacetotal1: "1000.0", diskspace2: "80.0", diskspacetotal2: "1000.0", slots: [{ nzo_id: "a", filename: "Waiting.One", status: "Queued", mb: "100", mbleft: "100", percentage: "0" }] } });
|
||||
if (mode === "history") return historyStatus === 200 ? send(200, { history: { slots: [{ nzo_id: "f", name: "Broken.Release", status: "Failed", fail_message: "Unpacking failed", completed: Math.floor(Date.now() / 1000) - 600 }] } }) : send(historyStatus, {});
|
||||
if (mode === "warnings") return send(200, { warnings: ["Failed to connect to news.example.com"] });
|
||||
send(200, {});
|
||||
});
|
||||
await new Promise<void>((resolve) => fake.listen(0, "127.0.0.1", resolve));
|
||||
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => fake.close());
|
||||
|
||||
test("the queue, what failed, the warnings and the disk — each read with the key, and nothing is asked that changes anything", async () => {
|
||||
seen.length = 0;
|
||||
const r = await adapter.createDownloadAdapter("sabnzbd", { url, apiKey: "good" }).read();
|
||||
assert.deepEqual([r.version, r.paused, r.speedDown, r.speedUp, r.connection, r.total, r.items.map((i) => i.name)], ["4.3.2", true, 0, null, null, 1, ["Waiting.One"]]);
|
||||
assert.deepEqual([r.failed?.map((f) => [f.name, f.message]), r.warnings?.count, r.disks?.length], [[["Broken.Release", "Unpacking failed"]], 1, 1]);
|
||||
assert.deepEqual(seen.map((s) => s.mode).sort(), ["history", "queue", "warnings"]);
|
||||
assert.ok(seen.every((s) => s.key === "good"));
|
||||
});
|
||||
|
||||
test("a history that cannot be read is null, not empty; the queue is what must work", async () => {
|
||||
historyStatus = 500;
|
||||
const r = await adapter.createDownloadAdapter("sabnzbd", { url, apiKey: "good" }).read();
|
||||
assert.deepEqual([r.failed, r.total], [null, 1]);
|
||||
historyStatus = 200;
|
||||
});
|
||||
|
||||
test("a key it refuses (answered with 200 and a status of false) and an address that is not SABnzbd are said", async () => {
|
||||
const bad = await adapter.createDownloadAdapter("sabnzbd", { url, apiKey: "wrong" }).ping();
|
||||
assert.equal(bad.ok, false);
|
||||
assert.match(bad.error ?? "", /SABnzbd did not accept the API key.*API Key Incorrect.*not the NZB key/);
|
||||
assert.equal((await adapter.createDownloadAdapter("sabnzbd", { url, apiKey: "good" }).ping()).ok, true);
|
||||
await assert.rejects(() => adapter.createDownloadAdapter("sabnzbd", { url: `${url}/nowhere`, apiKey: "good" }).read(), /no API at this address/);
|
||||
assert.equal((await adapter.createDownloadAdapter("sabnzbd", { url: "http://127.0.0.1:1", apiKey: "k" }).ping()).ok, false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("adding one", () => {
|
||||
test("qBittorrent needs only its address (a user name and password are for one that asks for them, and the password is kept secret); SABnzbd needs its API key", () => {
|
||||
assert.deepEqual(validateIntegrationConfig("qbittorrent", { url: "http://qb:8080" }), []);
|
||||
assert.deepEqual(validateIntegrationConfig("qbittorrent", {}), ["url"]);
|
||||
assert.deepEqual(validateIntegrationConfig("sabnzbd", { url: "http://sab:8080" }), ["apiKey"]);
|
||||
assert.deepEqual(validateIntegrationConfig("sabnzbd", { url: "http://sab:8080", apiKey: "k" }), []);
|
||||
const { secretFields, nonSecretFields } = splitIntegrationConfig("qbittorrent", { url: "http://qb:8080", username: "admin", password: "pw", insecure: true });
|
||||
assert.deepEqual(secretFields, { password: "pw" });
|
||||
assert.deepEqual(nonSecretFields, { url: "http://qb:8080", username: "admin", insecure: true });
|
||||
assert.equal(INTEGRATION_FIELDS.sabnzbd!.find((f) => f.key === "apiKey")!.secret, true);
|
||||
});
|
||||
|
||||
test("both kinds get the shared adapter, from the registry, with its two calls", () => {
|
||||
for (const kind of adapter.DOWNLOAD_KINDS) {
|
||||
const a = createIntegrationAdapter(kind, { url: "http://x" } as never) as unknown as Record<string, unknown>;
|
||||
assert.deepEqual([typeof a.ping, typeof a.read], ["function", "function"], kind);
|
||||
}
|
||||
assert.deepEqual(adapter.DOWNLOAD_KINDS, ["qbittorrent", "sabnzbd"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is wrong", () => {
|
||||
const NOW = Date.parse("2026-06-10T12:00:00Z");
|
||||
/** A stored state with nothing wrong, to change what a test is about. */
|
||||
const state = (extra: Partial<import("../src/services/downloadsSync.js").StoredDownload> = {}): import("../src/services/downloadsSync.js").StoredDownload => ({
|
||||
...read([item({ activityAt: iso(NOW - MIN) })], {}),
|
||||
since: { pausedSince: null, disconnectedSince: null, idleSince: null },
|
||||
updatedAt: iso(NOW),
|
||||
...extra,
|
||||
});
|
||||
const keys = (list: ReturnType<typeof dl.downloadProblems>) => list.map((p) => `${p.key}=${p.severity}`);
|
||||
const stalled = (extra: Partial<Item> = {}) => item({ id: "s", name: "Stalled.One", state: "stalled", rawState: "stalledDL", speed: 0, activityAt: iso(NOW - 2 * DAY), ...extra });
|
||||
|
||||
test("a client that is well says nothing", () => {
|
||||
assert.deepEqual(dl.downloadProblems("qbittorrent", 1, "Torrents", state(), NOW), []);
|
||||
assert.deepEqual(dl.downloadProblems("sabnzbd", 2, "Usenet", state({ speedDown: 0, counts: { ...adapter.emptyCounts() }, total: 0, items: [] }), NOW), []);
|
||||
});
|
||||
|
||||
describe("stuck downloads", () => {
|
||||
test("an error is stuck at once; stalled is stuck after a day without activity, not before; a download that moves is not", () => {
|
||||
assert.equal(dl.isStuck(item({ state: "error", rawState: "error" }), NOW), true);
|
||||
assert.equal(dl.isStuck(stalled(), NOW), true);
|
||||
assert.equal(dl.isStuck(stalled({ activityAt: iso(NOW - 5 * HOUR) }), NOW), false);
|
||||
assert.equal(dl.isStuck(item({ state: "downloading", activityAt: iso(NOW - 9 * DAY) }), NOW), false);
|
||||
assert.equal(dl.isStuck(stalled({ activityAt: null, addedAt: iso(NOW - 3 * DAY) }), NOW), true, "added long ago, never any activity");
|
||||
assert.equal(dl.isStuck(stalled({ activityAt: null, addedAt: null }), NOW), false, "one with no time cannot be called old");
|
||||
});
|
||||
test("one alert for the lot: how many, the first three by name, and why for the worst — an error first", () => {
|
||||
const items = [stalled({ id: "1", name: "A" }), item({ id: "2", name: "B", state: "error", rawState: "missingFiles" }), stalled({ id: "3", name: "C", activityAt: iso(NOW - 5 * DAY) }), stalled({ id: "4", name: "D" }), item({ id: "5", name: "fine" })];
|
||||
const p = dl.downloadProblems("qbittorrent", 1, "Torrents", state({ items, total: 5, counts: { ...adapter.emptyCounts(), stalled: 3, error: 1, downloading: 1 } }), NOW);
|
||||
assert.deepEqual(keys(p), ["qbittorrent-stuck:1=warning"]);
|
||||
assert.equal(p[0].message, "Torrents: 4 downloads are stuck — B; C; A; and 1 more (its files are missing)");
|
||||
const one = dl.downloadProblems("qbittorrent", 1, "Torrents", state({ items: [stalled({ name: "A" })] }), NOW);
|
||||
assert.equal(one[0].message, "Torrents: 1 download is stuck — A (nothing has moved for 2 days)");
|
||||
});
|
||||
});
|
||||
|
||||
describe("no connection", () => {
|
||||
test("disconnected for a quarter of an hour, with torrents, is raised; less is not; firewalled never is", () => {
|
||||
const at = (ago: number) => state({ connection: "disconnected", since: { pausedSince: null, disconnectedSince: iso(NOW - ago), idleSince: null } });
|
||||
assert.deepEqual(keys(dl.downloadProblems("qbittorrent", 1, "T", at(20 * MIN), NOW)), ["qbittorrent-connection:1=warning"]);
|
||||
assert.match(dl.downloadProblems("qbittorrent", 1, "T", at(2 * HOUR), NOW)[0].message, /^T: qBittorrent has had no connection to the network for 2 hours — if it is bound to a VPN/);
|
||||
assert.deepEqual(dl.downloadProblems("qbittorrent", 1, "T", at(5 * MIN), NOW), []);
|
||||
assert.deepEqual(dl.downloadProblems("qbittorrent", 1, "T", { ...at(2 * HOUR), connection: "firewalled" }, NOW), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a queue left paused, and one getting nothing", () => {
|
||||
const sabCounts = { ...adapter.emptyCounts(), queued: 2 };
|
||||
test("paused for six hours with downloads waiting is raised; paused with nothing waiting, or for less, is not", () => {
|
||||
const paused = (ago: number, total = 2) => state({ paused: true, speedDown: 0, total, counts: total ? sabCounts : adapter.emptyCounts(), since: { pausedSince: iso(NOW - ago), disconnectedSince: null, idleSince: null } });
|
||||
const p = dl.downloadProblems("sabnzbd", 2, "Usenet", paused(7 * HOUR), NOW);
|
||||
assert.deepEqual(keys(p), ["sabnzbd-paused:2=warning"]);
|
||||
assert.equal(p[0].message, "Usenet: SABnzbd has been paused for 7 hours with 2 downloads waiting");
|
||||
assert.deepEqual(dl.downloadProblems("sabnzbd", 2, "Usenet", paused(2 * HOUR), NOW), []);
|
||||
assert.deepEqual(dl.downloadProblems("sabnzbd", 2, "Usenet", paused(9 * HOUR, 0), NOW), []);
|
||||
});
|
||||
test("work waiting and nothing coming in for an hour is raised, and says how much waits; a paused queue is the other alert's business", () => {
|
||||
const idle = (ago: number, extra: Partial<import("../src/services/downloadsSync.js").StoredDownload> = {}) => state({ speedDown: 0, total: 2, counts: sabCounts, since: { pausedSince: null, disconnectedSince: null, idleSince: iso(NOW - ago) }, ...extra });
|
||||
const p = dl.downloadProblems("sabnzbd", 2, "Usenet", idle(3 * HOUR), NOW);
|
||||
assert.deepEqual(keys(p), ["sabnzbd-stalled:2=warning"]);
|
||||
assert.equal(p[0].message, "Usenet: SABnzbd has downloaded nothing for 3 hours although 2 downloads are waiting — can it reach its news server?");
|
||||
assert.deepEqual(dl.downloadProblems("sabnzbd", 2, "Usenet", idle(20 * MIN), NOW), []);
|
||||
assert.deepEqual(dl.downloadProblems("sabnzbd", 2, "Usenet", idle(3 * HOUR, { paused: true }), NOW).filter((x) => x.key.includes("stalled")), []);
|
||||
});
|
||||
test("what is seen between reads: the clock starts at the first read that shows it, is kept while it goes on, and is dropped when it is over", () => {
|
||||
const t0 = new Date(NOW);
|
||||
const t1 = new Date(NOW + 10 * MIN);
|
||||
const waiting = { total: 2, counts: sabCounts };
|
||||
const first = dl.carrySince({ paused: false, connection: null, speedDown: 0, ...waiting }, null, t0);
|
||||
assert.deepEqual(first, { pausedSince: null, disconnectedSince: null, idleSince: t0.toISOString() });
|
||||
assert.equal(dl.carrySince({ paused: false, connection: null, speedDown: 0, ...waiting }, first, t1).idleSince, t0.toISOString());
|
||||
assert.equal(dl.carrySince({ paused: false, connection: null, speedDown: 500, ...waiting }, first, t1).idleSince, null);
|
||||
assert.equal(dl.carrySince({ paused: true, connection: null, speedDown: 0, ...waiting }, first, t1).pausedSince, t1.toISOString());
|
||||
assert.equal(dl.carrySince({ paused: false, connection: "disconnected", speedDown: 0, total: 0, counts: adapter.emptyCounts() }, null, t0).disconnectedSince, null, "nothing loaded, nothing to be cut off from");
|
||||
assert.equal(dl.carrySince({ paused: false, connection: "disconnected", speedDown: 0, total: 3, counts: adapter.emptyCounts() }, null, t0).disconnectedSince, t0.toISOString());
|
||||
});
|
||||
});
|
||||
|
||||
describe("disks", () => {
|
||||
test("where the size is said: under 10% free is a warning, under 5% critical, a tiny disk is not judged", () => {
|
||||
const d = (free: number, total = 1000) => state({ disks: [{ label: "Complete folder", free: free * GB, total: total * GB }] });
|
||||
assert.deepEqual(dl.downloadProblems("sabnzbd", 2, "U", d(150), NOW), []);
|
||||
const warn = dl.downloadProblems("sabnzbd", 2, "U", d(80), NOW);
|
||||
assert.deepEqual(keys(warn), ["sabnzbd-disk:2:complete-folder=warning"]);
|
||||
assert.equal(warn[0].message, "U: the complete folder of SABnzbd has 80.0 GB free of 1000 GB (8%)");
|
||||
assert.deepEqual(keys(dl.downloadProblems("sabnzbd", 2, "U", d(30), NOW)), ["sabnzbd-disk:2:complete-folder=critical"]);
|
||||
assert.deepEqual(dl.downloadProblems("sabnzbd", 2, "U", state({ disks: [{ label: "Tmp", free: 0, total: GB }] }), NOW), []);
|
||||
});
|
||||
test("where it is not (qBittorrent says only what is free): under 20 GB is a warning, under 5 GB critical; a disk that was not read says nothing", () => {
|
||||
const d = (free: number) => state({ disks: [{ label: "Download folder", free: free * GB, total: null }] });
|
||||
assert.deepEqual(dl.downloadProblems("qbittorrent", 1, "T", d(200), NOW), []);
|
||||
assert.equal(dl.downloadProblems("qbittorrent", 1, "T", d(12), NOW)[0].message, "T: the download folder of qBittorrent has only 12.0 GB free");
|
||||
assert.deepEqual(keys(dl.downloadProblems("qbittorrent", 1, "T", d(3), NOW)), ["qbittorrent-disk:1:download-folder=critical"]);
|
||||
assert.deepEqual(dl.downloadProblems("qbittorrent", 1, "T", state({ disks: null }), NOW), []);
|
||||
});
|
||||
});
|
||||
|
||||
test("what failed in the last day is news; older is not", () => {
|
||||
const failed = [item({ id: "n", state: "failed", activityAt: iso(NOW - 2 * HOUR) }), item({ id: "o", state: "failed", activityAt: iso(NOW - 3 * DAY) }), item({ id: "x", state: "failed", activityAt: null })];
|
||||
assert.deepEqual(dl.recentFailed(failed, NOW).map((f) => f.id), ["n"]);
|
||||
assert.deepEqual(dl.recentFailed(null, NOW), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is stored", () => {
|
||||
let id = 0;
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "sabnzbd", name: "Usenet", baseUrl: "http://sab", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
});
|
||||
const row = async () => (await db.select().from(downloadState).where(eq(downloadState.integrationId, id)))[0];
|
||||
after(async () => {
|
||||
await dl.forgetDownload(id);
|
||||
await db.delete(integrations).where(eq(integrations.id, id));
|
||||
});
|
||||
|
||||
test("the state is replaced at each read; a part that could not be read is stored as null", async () => {
|
||||
await dl.storeRead(id, read([item()], { disks: null, failed: null }), new Date());
|
||||
let s = dl.storedOf(await row());
|
||||
assert.deepEqual([s.disks, s.failed, s.total, s.items.length, s.counts.downloading], [null, null, 1, 1, 1]);
|
||||
await dl.storeRead(id, read([item(), item({ id: "h2", state: "queued" })], { failed: [item({ id: "f", state: "failed" })] }), new Date());
|
||||
s = dl.storedOf(await row());
|
||||
assert.deepEqual([s.total, s.counts.queued, s.failed?.length], [2, 1, 1]);
|
||||
});
|
||||
|
||||
test("since when it has been so is carried from read to read, and dropped when it is over", async () => {
|
||||
const t0 = new Date(Date.now() - 8 * HOUR);
|
||||
const paused = (extra: Partial<Read> = {}) => read([item({ state: "queued" })], { paused: true, speedDown: 0, ...extra });
|
||||
await dl.storeRead(id, paused(), t0);
|
||||
await dl.storeRead(id, paused(), new Date());
|
||||
assert.equal(dl.storedOf(await row()).since.pausedSince, t0.toISOString());
|
||||
await dl.storeRead(id, read(), new Date());
|
||||
assert.equal(dl.storedOf(await row()).since.pausedSince, null);
|
||||
});
|
||||
|
||||
test("a read that failed is said, and what was stored is left alone", async () => {
|
||||
const ok = await dl.syncDownload(id, new Date(), async () => read([item()], { version: "kept" }));
|
||||
assert.equal(ok.ok, true);
|
||||
const failed = await dl.syncDownload(id, new Date(), async () => {
|
||||
throw new Error("connection refused");
|
||||
});
|
||||
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
|
||||
assert.equal(failed.at, ok.at);
|
||||
assert.equal(dl.storedOf(await row()).version, "kept");
|
||||
assert.equal(dl.isDownloadFresh((await dl.downloadSyncStatuses())[id], Date.now()), false);
|
||||
});
|
||||
|
||||
test("the kind is what the read is asked for, and an integration of another type is not read", async () => {
|
||||
const asked: string[] = [];
|
||||
await dl.syncDownload(id, new Date(), async (kind) => {
|
||||
asked.push(kind);
|
||||
return read();
|
||||
});
|
||||
assert.deepEqual(asked, ["sabnzbd"]);
|
||||
const other = (await db.insert(integrations).values({ type: "gitea", name: "G", baseUrl: "http://g", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
const status = await dl.syncDownload(other, new Date(), async () => read());
|
||||
assert.deepEqual([status.ok, status.error], [false, "not a qBittorrent or SABnzbd integration"]);
|
||||
await dl.forgetDownload(other);
|
||||
await db.delete(integrations).where(eq(integrations.id, other));
|
||||
});
|
||||
});
|
||||
|
||||
describe("the health pass, the page and /metrics", () => {
|
||||
let qId = 0;
|
||||
let sId = 0;
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const fresh = () => setInternalFlag("downloadsSync", JSON.stringify({ [qId]: { at: new Date().toISOString(), ok: true, error: null }, [sId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
const stale = () => setInternalFlag("downloadsSync", JSON.stringify({ [qId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null }, [sId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
const mk = async (type: "qbittorrent" | "sabnzbd", name: string) => (await db.insert(integrations).values({ type, name, baseUrl: `http://${name}`, config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
qId = await mk("qbittorrent", "Torrents");
|
||||
sId = await mk("sabnzbd", "Usenet");
|
||||
const now = Date.now();
|
||||
// Twenty minutes with no network, then still none; the client has an errored torrent, a stalled one, and 3 GB free.
|
||||
const torrents = [item({ id: "e", name: "Broken", state: "error", rawState: "missingFiles" }), item({ id: "s", name: "Old.Stall", state: "stalled", rawState: "stalledDL", speed: 0, activityAt: iso(now - 3 * DAY) }), item({ id: "d", name: "Moving", state: "downloading" }), item({ id: "u", name: "Seeded", state: "seeding" })];
|
||||
const q = (extra: Partial<Read> = {}) => read(torrents, { connection: "disconnected", disks: [{ label: "Download folder", free: 3 * GB, total: null }], ...extra });
|
||||
await dl.storeRead(qId, q(), new Date(now - 20 * MIN));
|
||||
await dl.storeRead(qId, q(), new Date(now));
|
||||
// A queue paused for seven hours with two waiting, and a nearly full disk; one failure a minute ago and one last week.
|
||||
const sab = (extra: Partial<Read> = {}) =>
|
||||
read([item({ id: "a", name: "Waiting.1", state: "queued" }), item({ id: "b", name: "Waiting.2", state: "queued" })], {
|
||||
paused: true,
|
||||
speedDown: 0,
|
||||
speedUp: null,
|
||||
connection: null,
|
||||
disks: [{ label: "Complete folder", free: 30 * GB, total: 1000 * GB }],
|
||||
failed: [item({ id: "f1", name: "Failed.Now", state: "failed", message: "Unpacking failed", activityAt: iso(now - MIN) }), item({ id: "f2", name: "Failed.Then", state: "failed", activityAt: iso(now - 7 * DAY) })],
|
||||
warnings: { count: 1, recent: ["Failed to connect to news.example.com"] },
|
||||
...extra,
|
||||
});
|
||||
await dl.storeRead(sId, sab(), new Date(now - 7 * HOUR));
|
||||
await dl.storeRead(sId, sab(), new Date(now));
|
||||
await fresh();
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/downloads", downloadsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/downloads`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("the health pass judges each from what is stored, and holds one whose last read is old", async () => {
|
||||
let { snapshot, held } = await collectSnapshot();
|
||||
assert.deepEqual(snapshot.downloads?.map((d) => [d.kind, d.integrationId]).sort(), [["qbittorrent", qId], ["sabnzbd", sId]].sort());
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const mine = list.filter((x) => /^(qbittorrent|sabnzbd)-/.test(x.key));
|
||||
assert.deepEqual(
|
||||
mine.map((x) => x.key).sort(),
|
||||
[`qbittorrent-connection:${qId}`, `qbittorrent-disk:${qId}:download-folder`, `qbittorrent-stuck:${qId}`, `sabnzbd-disk:${sId}:complete-folder`, `sabnzbd-paused:${sId}`].sort(),
|
||||
);
|
||||
const disk = mine.find((x) => x.key === `qbittorrent-disk:${qId}:download-folder`)!;
|
||||
assert.deepEqual([disk.severity, disk.source], ["critical", `qbittorrent:${qId}`]);
|
||||
assert.match(mine.find((x) => x.key === `qbittorrent-stuck:${qId}`)!.message, /2 downloads are stuck — Broken; Old\.Stall/);
|
||||
for (const k of ["stuck", "connection", "paused", "stalled", "disk"]) assert.equal(subjectOfConditionKey(`qbittorrent-${k}:${qId}:x`), `integration:${qId}`);
|
||||
await stale();
|
||||
({ snapshot, held } = await collectSnapshot());
|
||||
assert.deepEqual([snapshot.downloads?.map((d) => d.integrationId), held.has(`qbittorrent:${qId}`)], [[sId], true]);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("the Alerts page says where to look: the disk under Disk, the rest under Services, and the link is the download clients page", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const stuck = report.alerts.find((x) => x.id.endsWith(`qbittorrent-stuck:${qId}`))!;
|
||||
const disk = report.alerts.find((x) => x.id.endsWith(`sabnzbd-disk:${sId}:complete-folder`))!;
|
||||
const paused = report.alerts.find((x) => x.id.endsWith(`sabnzbd-paused:${sId}`))!;
|
||||
assert.deepEqual([stuck.category, stuck.source, stuck.link, stuck.target?.type, disk.category, disk.source, paused.category], ["services", "qBittorrent", "/downloads", "integration", "disk", "SABnzbd", "services"]);
|
||||
});
|
||||
|
||||
test("the page: each client with its kind, what is wrong, its items (the stuck ones marked), what failed in the last day, its disks and its own warnings", async () => {
|
||||
type Body = { clients: { id: number; kind: string; fresh: boolean; problems: { key: string }[]; state: { stuck: string[]; failedRecent: string[]; total: number; items: { name: string }[]; connection: string | null; paused: boolean; warnings: { count: number } | null; failed: unknown[] | null } }[] };
|
||||
const body = (await (await fetch(base)).json()) as Body;
|
||||
const q = body.clients.find((c) => c.id === qId)!;
|
||||
assert.deepEqual([q.kind, q.fresh, q.state.total, q.state.items.map((i) => i.name), q.state.stuck.sort(), q.state.connection, q.problems.length], ["qbittorrent", true, 4, ["Broken", "Old.Stall", "Moving"], ["e", "s"], "disconnected", 3]);
|
||||
const s = body.clients.find((c) => c.id === sId)!;
|
||||
assert.deepEqual([s.kind, s.state.paused, s.state.failed?.length, s.state.failedRecent, s.state.warnings?.count, s.problems.length], ["sabnzbd", true, 2, ["f1"], 1, 2]);
|
||||
await stale();
|
||||
const old = (await (await fetch(base)).json()) as Body;
|
||||
assert.deepEqual(old.clients.find((c) => c.id === qId)!.problems, [], "an old reading says nothing is wrong");
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("/metrics has the speeds, the items in each state, what is stuck, paused, connected, failed and the free space — only while the reading is recent", async () => {
|
||||
let text = await buildMetrics(new Date());
|
||||
assert.match(text, /homelab_download_speed_bytes\{client="qbittorrent",name="Torrents",direction="down"\} 1000/);
|
||||
assert.match(text, /homelab_download_speed_bytes\{client="qbittorrent",name="Torrents",direction="up"\} 10/);
|
||||
assert.doesNotMatch(text, /homelab_download_speed_bytes\{client="sabnzbd",name="Usenet",direction="up"\}/, "SABnzbd does not say");
|
||||
assert.match(text, /homelab_download_items\{client="qbittorrent",name="Torrents",state="error"\} 1/);
|
||||
assert.match(text, /homelab_download_items\{client="sabnzbd",name="Usenet",state="queued"\} 2/);
|
||||
assert.match(text, /homelab_download_stuck_items\{client="qbittorrent",name="Torrents"\} 2/);
|
||||
assert.match(text, /homelab_download_paused\{client="sabnzbd",name="Usenet"\} 1/);
|
||||
assert.match(text, /homelab_download_connected\{client="qbittorrent",name="Torrents"\} 0/);
|
||||
assert.doesNotMatch(text, /homelab_download_connected\{client="sabnzbd"/, "SABnzbd has no such thing");
|
||||
assert.match(text, /homelab_download_failed_recent\{client="sabnzbd",name="Usenet"\} 1/);
|
||||
assert.match(text, /homelab_download_disk_free_bytes\{client="qbittorrent",name="Torrents",folder="Download folder"\} 3221225472/);
|
||||
await stale();
|
||||
text = await buildMetrics(new Date());
|
||||
assert.doesNotMatch(text, /homelab_download_[a-z_]+\{client="qbittorrent"/);
|
||||
assert.match(text, /homelab_download_paused\{client="sabnzbd"/, "the other, still fresh, stays");
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("a wrong kind of row is no download client, and taking the integration away takes what was read with it", async () => {
|
||||
await dl.forgetDownload(qId);
|
||||
assert.equal((await dl.downloadSyncStatuses())[qId], undefined);
|
||||
await db.delete(integrations).where(eq(integrations.id, qId));
|
||||
assert.deepEqual(await db.select().from(downloadState).where(eq(downloadState.integrationId, qId)), []);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user