Sonarr/Radarr/Prowlarr, Hetzner load balancers/floating IPs/certificates, traffic forecast

New integration kinds sonarr, radarr and prowlarr (shared adapter and sync, page under
Apps -> Media managers): own health findings, stuck downloads, nearly full disks.
Hetzner now reads load balancers (target health), floating IPs and certificates, includes
them in the cost estimate, and forecasts the month's traffic against the quota.
Migrations 0072 and 0073; docs, metrics and tests included.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-27 20:04:55 +01:00
1 parent 9a3d5c466a
commit d9083adb5b
38 files changed
+18374 -42

No files matched your search

+3
View File
@@ -0,0 +1,3 @@
ALTER TABLE `hetzner_state` ADD `load_balancers` text;--> statement-breakpoint
ALTER TABLE `hetzner_state` ADD `floating_ips` text;--> statement-breakpoint
ALTER TABLE `hetzner_state` ADD `certificates` text;
@@ -0,0 +1,11 @@
CREATE TABLE `arr_state` (
`integration_id` integer PRIMARY KEY NOT NULL,
`version` text,
`health` text DEFAULT '[]' NOT NULL,
`queue` text,
`disks` text,
`missing` integer,
`indexers` text,
`updated_at` text NOT NULL,
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
);
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+14
View File
@@ -505,6 +505,20 @@
"when": 1793122879270,
"tag": "0071_cuddly_strong_guy",
"breakpoints": true
},
{
"idx": 72,
"version": "6",
"when": 1793126445007,
"tag": "0072_fair_shiver_man",
"breakpoints": true
},
{
"idx": 73,
"version": "6",
"when": 1793126767536,
"tag": "0073_chilly_silver_surfer",
"breakpoints": true
}
]
}
+2
View File
@@ -69,6 +69,7 @@ import { netboxRouter } from "./routes/netbox.js";
import { hetznerRouter } from "./routes/hetzner.js";
import { speedtestRouter } from "./routes/speedtest.js";
import { fragnetRouter } from "./routes/fragnet.js";
import { arrRouter } from "./routes/arr.js";
import { costsRouter } from "./routes/costs.js";
import { nextcloudRouter } from "./routes/nextcloud.js";
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
@@ -216,6 +217,7 @@ export function createApp() {
app.use("/api/hetzner", hetznerRouter);
app.use("/api/speedtest", speedtestRouter);
app.use("/api/fragnet", fragnetRouter);
app.use("/api/arr", arrRouter);
app.use("/api/costs", costsRouter);
app.use("/api/nextcloud", nextcloudRouter);
app.use("/api/service-levels", serviceLevelsRouter);
+24
View File
@@ -564,6 +564,9 @@ export const integrationTypes = [
"hetzner",
"speedtest",
"fragnet",
"sonarr",
"radarr",
"prowlarr",
] as const;
export type IntegrationType = (typeof integrationTypes)[number];
@@ -1270,6 +1273,9 @@ export const hetznerState = sqliteTable("hetzner_state", {
servers: text("servers").notNull().default("[]"), // JSON: each server as read, plus when it was last seen running, since when it has had this status and since when rescue mode has been on
volumes: text("volumes"), // JSON; null: could not be asked
images: text("images"), // JSON: snapshots and backups; null: could not be asked
loadBalancers: text("load_balancers"), // JSON: each load balancer with the health of its targets; null: could not be asked
floatingIps: text("floating_ips"), // JSON; null: could not be asked
certificates: text("certificates"), // JSON: each certificate with when it ends; null: could not be asked
cost: text("cost"), // JSON: the monthly estimate (currency, servers, backups, volumes, snapshots, IPs, total); null: could not be worked out (no price list)
updatedAt: text("updated_at").notNull(), // ISO
});
@@ -1286,6 +1292,24 @@ export const fragnetState = sqliteTable("fragnet_state", {
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── Sonarr, Radarr, Prowlarr ──────────────────────────────────────────────
// How a Sonarr, Radarr or Prowlarr (an integration of type "sonarr", "radarr" or "prowlarr") is right now, one row each, replaced at every read: what it says is wrong with itself, its download
// queue (the first page, and how many there are), its disks, how many wanted items are missing and — Prowlarr — its indexers. A column that is null could not be read, which is not the same as
// none. Read by services/arrSync.ts.
export const arrState = sqliteTable("arr_state", {
integrationId: integer("integration_id")
.primaryKey()
.references(() => integrations.id, { onDelete: "cascade" }),
version: text("version"),
health: text("health").notNull().default("[]"), // JSON: what its own checks found
queue: text("queue"), // JSON { total, items }; null: could not be read (and always null for Prowlarr)
disks: text("disks"), // JSON; null: could not be read
missing: integer("missing"), // null: not read (Prowlarr has none)
indexers: text("indexers"), // JSON, Prowlarr only; null otherwise or when it could not be read
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── NetBox ─────────────────────────────────────────────────────────────────
// How a NetBox (an integration of type "netbox") is right now, one row each, replaced at every read: its version, how many background workers are running, the counts and the IP
+2
View File
@@ -41,6 +41,7 @@ import { initDockerVulnSync } from "./services/dockerVulnSync.js";
import { initHetznerSync } from "./services/hetznerSync.js";
import { initSpeedtestSync } from "./services/speedtestSync.js";
import { initFragnetSync } from "./services/fragnetSync.js";
import { initArrSync } from "./services/arrSync.js";
import { backfillMetricRollups } from "./services/serverMetrics.js";
import { backfillEndpointHourly } from "./services/endpointChecks.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
@@ -103,6 +104,7 @@ initDockerVulnSync();
initHetznerSync();
initSpeedtestSync();
initFragnetSync();
initArrSync();
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
+234
View File
@@ -0,0 +1,234 @@
/**
* Sonarr, Radarr and Prowlarr adapter — the three speak the same API (the "*arr" family), so one adapter serves all of them, told which it is. Config: url, apiKey (the application's own: Settings →
* General → Security → API Key; sent as `X-Api-Key`) and optionally insecure.
*
* Reading only: what the application says is wrong with itself (`/health`), its download queue (`/queue`: what is downloading, what is stuck or failed and why), the disks it uses
* (`/diskspace`), how many wanted items are missing (`/wanted/missing`, Sonarr and Radarr) and — for Prowlarr — its indexers and which of them it has switched off after failures (`/indexer`,
* `/indexerstatus`). Nothing is sent but GET requests: no search is started, nothing is grabbed, removed or changed.
*
* Sonarr and Radarr answer under `/api/v3`, Prowlarr under `/api/v1`. Endpoints and field names are from their published OpenAPI files and source (HealthResource, QueueResource,
* DiskSpaceResource, IndexerStatusResource) and have not been checked against live installs here; every field is read defensively.
*/
import { requestJson } from "../httpJson.js";
import { withDiagLogging } from "../../services/diagLog.js";
export type ArrKind = "sonarr" | "radarr" | "prowlarr";
export const ARR_KINDS: ArrKind[] = ["sonarr", "radarr", "prowlarr"];
export const ARR_NAMES: Record<ArrKind, string> = { sonarr: "Sonarr", radarr: "Radarr", prowlarr: "Prowlarr" };
export interface ArrConfig {
url: string;
apiKey: string;
insecure?: boolean;
}
export interface ArrHealthIssue {
/** The check that found it ("IndexerStatusCheck", "DownloadClientCheck" …). */
source: string;
/** ok, notice, warning or error. */
type: string;
message: string;
}
export interface ArrQueueItem {
id: number;
title: string;
/** The application's own status of the item: downloading, paused, completed, warning, failed … (lower case). */
status: string;
/** How the download is judged: ok, warning or error; null when not said. */
tracked: "ok" | "warning" | "error" | null;
/** downloading, importPending, importBlocked, failedPending, failed, ignored …; null when not said. */
state: string | null;
/** What it says is wrong (the status messages and the error), when anything. */
messages: string[];
size: number | null;
sizeLeft: number | null;
added: string | null;
etaAt: string | null;
protocol: string | null;
client: string | null;
indexer: string | null;
}
export interface ArrDisk {
path: string;
label: string | null;
free: number;
total: number;
}
export interface ArrIndexer {
id: number;
name: string;
enabled: boolean;
protocol: string | null;
/** When Prowlarr will try it again, if it has switched it off after failures. */
disabledTill: string | null;
lastFailure: string | null;
}
export interface ArrRead {
version: string | null;
health: ArrHealthIssue[];
/** Null when the queue could not be read. `total` is what the application says there are, `items` the first page of them. */
queue: { total: number; items: ArrQueueItem[] } | null;
disks: ArrDisk[] | null;
/** Sonarr and Radarr: how many wanted items have no file yet; null when not read. */
missing: number | null;
/** Prowlarr only. */
indexers: ArrIndexer[] | null;
}
export interface ArrAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
read(): Promise<ArrRead>;
}
export const QUEUE_PAGE = 100;
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
const arr = (v: unknown): unknown[] => (Array.isArray(v) ? v : []);
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
const iso = (v: unknown): string | null => {
const s = str(v);
return s !== null && Number.isFinite(Date.parse(s)) ? new Date(s).toISOString() : null;
};
/** Pure: `/health` — what each check found; anything that is not a list is no findings. */
export function parseHealth(raw: unknown): ArrHealthIssue[] {
const out: ArrHealthIssue[] = [];
for (const item of arr(raw)) {
const h = obj(item);
const message = str(h.message);
if (!message) continue;
out.push({ source: str(h.source) ?? "Health", type: (str(h.type) ?? "warning").toLowerCase(), message });
}
return out;
}
/** Pure: one item of the queue. */
export function parseQueueItem(raw: unknown): ArrQueueItem | null {
const q = obj(raw);
const id = num(q.id);
if (id === null) return null;
const tracked = str(q.trackedDownloadStatus)?.toLowerCase();
const messages: string[] = [];
for (const m of arr(q.statusMessages)) {
const title = str(obj(m).title);
const lines = arr(obj(m).messages).map((x) => str(x)).filter((x): x is string => x !== null);
if (lines.length > 0) messages.push(...lines.map((l) => (title ? `${title}: ${l}` : l)));
else if (title) messages.push(title);
}
const error = str(q.errorMessage);
if (error) messages.unshift(error);
return {
id,
title: str(q.title) ?? `#${id}`,
status: (str(q.status) ?? "unknown").toLowerCase(),
tracked: tracked === "ok" || tracked === "warning" || tracked === "error" ? tracked : null,
state: str(q.trackedDownloadState),
messages: [...new Set(messages)].slice(0, 4),
size: num(q.size),
sizeLeft: num(q.sizeleft),
added: iso(q.added),
etaAt: iso(q.estimatedCompletionTime),
protocol: str(q.protocol),
client: str(q.downloadClient),
indexer: str(q.indexer),
};
}
/** Pure: a page of `/queue` (`records`, and the total the application says there are). */
export function parseQueue(raw: unknown): { total: number; items: ArrQueueItem[] } {
const q = obj(raw);
const items = arr(q.records).map(parseQueueItem).filter((i): i is ArrQueueItem => i !== null);
return { total: num(q.totalRecords) ?? items.length, items };
}
/** Pure: `/diskspace`. */
export function parseDisks(raw: unknown): ArrDisk[] {
const out: ArrDisk[] = [];
for (const item of arr(raw)) {
const d = obj(item);
const path = str(d.path);
const free = num(d.freeSpace);
const total = num(d.totalSpace);
if (!path || free === null || total === null) continue;
out.push({ path, label: str(d.label), free, total });
}
return out;
}
/** Pure: Prowlarr's indexers with what `/indexerstatus` says of each. */
export function parseIndexers(indexers: unknown, status: unknown): ArrIndexer[] {
const failing = new Map<number, Record<string, unknown>>();
for (const s of arr(status)) {
const o = obj(s);
const id = num(o.indexerId);
if (id !== null) failing.set(id, o);
}
const out: ArrIndexer[] = [];
for (const item of arr(indexers)) {
const i = obj(item);
const id = num(i.id);
const name = str(i.name);
if (id === null || !name) continue;
const st = failing.get(id);
out.push({ id, name, enabled: i.enable !== false, protocol: str(i.protocol), disabledTill: iso(st?.disabledTill), lastFailure: iso(st?.mostRecentFailure) });
}
return out.sort((a, b) => a.name.localeCompare(b.name));
}
export function createArrAdapter(kind: ArrKind, config: ArrConfig): ArrAdapter {
const base = `${String(config.url ?? "").trim().replace(/\/+$/, "")}/api/${kind === "prowlarr" ? "v1" : "v3"}`;
const insecure = config.insecure === true;
const headers = { "X-Api-Key": String(config.apiKey ?? "").trim(), Accept: "application/json" };
const name = ARR_NAMES[kind];
async function get(path: string): Promise<unknown> {
const res = await requestJson(`${base}${path}`, { headers, insecure });
if (res.status === 401 || res.status === 403) throw new Error(`${name} did not accept the API key (HTTP ${res.status}) — copy it again from Settings → General → Security → API Key`);
if (res.status === 404) throw new Error(`${name} has no API at this address (HTTP 404) — check the address (its own, not a proxy's), and that it is ${name}`);
if (res.status < 200 || res.status >= 300) throw new Error(`${name} error: HTTP ${res.status}`);
if (res.json === null) throw new Error(`${name} answered with something that isn't JSON — is the address its own?`);
return res.json;
}
const soft = async <T>(run: () => Promise<T>): Promise<T | null> => {
try {
return await run();
} catch {
return null;
}
};
async function read(): Promise<ArrRead> {
// The status and the health must work: they are what says it is there and what is wrong.
const status = obj(await get("/system/status"));
const health = parseHealth(await get("/health"));
const include = kind === "sonarr" ? "&includeUnknownSeriesItems=true" : kind === "radarr" ? "&includeUnknownMovieItems=true" : "";
const [queue, disks, missing, indexers, indexerStatus] = await Promise.all([
kind === "prowlarr" ? Promise.resolve(null) : soft(async () => parseQueue(await get(`/queue?page=1&pageSize=${QUEUE_PAGE}${include}`))),
soft(async () => parseDisks(await get("/diskspace"))),
kind === "prowlarr" ? Promise.resolve(null) : soft(async () => num(obj(await get("/wanted/missing?page=1&pageSize=1&monitored=true")).totalRecords)),
kind === "prowlarr" ? soft(() => get("/indexer")) : Promise.resolve(null),
kind === "prowlarr" ? soft(() => get("/indexerstatus")) : Promise.resolve(null),
]);
return { version: str(status.version), health, queue, disks, missing, indexers: kind === "prowlarr" && indexers !== null ? parseIndexers(indexers, indexerStatus) : null };
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
const start = Date.now();
try {
const status = obj(await get("/system/status"));
const app = str(status.appName);
if (app && app.toLowerCase() !== kind) throw new Error(`That is ${app}, not ${name} — add it as the kind it is`);
return { ok: true, latencyMs: Date.now() - start };
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) };
}
}
return withDiagLogging(kind, { ping, read });
}
+15
View File
@@ -114,6 +114,21 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
hetzner: [{ key: "token", label: "API token", secret: true, type: "password", placeholder: "Hetzner Console → the project → Security → API tokens; Read is enough" }],
sonarr: [
{ key: "url", label: "Sonarr URL", secret: false, placeholder: "http://sonarr.example.lan:8989" },
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "Settings → General → Security → API Key" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
radarr: [
{ key: "url", label: "Radarr URL", secret: false, placeholder: "http://radarr.example.lan:7878" },
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "Settings → General → Security → API Key" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
prowlarr: [
{ key: "url", label: "Prowlarr URL", secret: false, placeholder: "http://prowlarr.example.lan:9696" },
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "Settings → General → Security → API Key" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
fragnet: [
{ key: "token", label: "API key", secret: true, type: "password", placeholder: "ptlc_… — the game panel: profile → API Credentials (not an Application key, ptla_)" },
{ key: "url", label: "Panel address (optional)", secret: false, optional: true, placeholder: "https://panel.fragnet.net — only if yours is somewhere else" },
+178 -4
View File
@@ -2,8 +2,8 @@
* Hetzner Cloud adapter. Config: token (a project's API token — Hetzner Console → the project → Security → API tokens; **Read** is enough).
*
* Reading only: the project's servers (status, type, location, addresses, traffic used and included this month, whether rescue mode is on and whether backups are enabled), its volumes, its
* snapshots and backups, its primary IPs and Hetzner's price list — enough to say what is off, what is close to its traffic quota, what has no recent backup, and roughly what the project
* costs. Nothing is sent to Hetzner but GET requests for these; no server is started, stopped or changed.
* snapshots and backups, its primary and floating IPs, its load balancers (and the health of their targets), its certificates (and when they expire) and Hetzner's price list — enough to say
* what is off, what is close to its traffic quota, what has no recent backup, what is unhealthy or about to expire, and roughly what the project costs. Nothing is sent to Hetzner but GET requests for these; no server is started, stopped or changed.
*
* Endpoints and field names are from Hetzner's published OpenAPI file (`/servers`, `/volumes`, `/images`, `/primary_ips`, `/pricing`; lists are paged, 50 at a time) and have not been
* checked against a live project here; every field is read defensively. A token belongs to one project, so one integration is one project. The API allows 3600 requests an hour per project.
@@ -75,8 +75,65 @@ export interface HetznerPrimaryIp {
assigned: boolean;
}
/** One thing a load balancer sends to: a server (by id), every server with a label, or an address. */
export interface HetznerLbTarget {
kind: "server" | "label_selector" | "ip";
/** The Hetzner server, for kind "server" (and for what a label selector found). */
serverId: number | null;
/** The label selector, or the address, for the other two. */
label: string | null;
/** unhealthy as soon as one of its ports is; healthy when all are; unknown when Hetzner has not judged yet. */
health: "healthy" | "unhealthy" | "unknown";
}
export interface HetznerLoadBalancer {
id: number;
name: string;
type: string | null;
location: string | null;
ipv4: string | null;
ipv6: string | null;
algorithm: string | null;
/** How many services (listening ports) it has. */
services: number;
targets: HetznerLbTarget[];
includedTraffic: number | null;
outgoingTraffic: number | null;
/** The price of its type in its location per month, including VAT; null when not said. */
monthlyPrice: number | null;
}
export interface HetznerFloatingIp {
id: number;
name: string | null;
ip: string;
type: "ipv4" | "ipv6";
/** The server it points at; null when it points at none (it is billed all the same). */
serverId: number | null;
location: string | null;
description: string | null;
}
export interface HetznerCertificate {
id: number;
name: string;
/** uploaded or managed (Hetzner gets and renews it itself). */
type: string;
domains: string[];
/** ISO; null while a managed one is still being issued. */
notValidAfter: string | null;
/** For a managed certificate: pending, completed or failed — and the same for the renewal. */
issuance: string | null;
renewal: string | null;
error: string | null;
/** How many resources (load balancers) use it. */
usedBy: number;
}
export interface HetznerPrices {
currency: string;
/** Per month, including VAT, for each kind and location of floating IP; a location of "*" is for every location. */
floatingIps: { type: string; location: string; monthly: number }[];
/** Per GB and month, including VAT. */
imagePerGb: number | null;
volumePerGb: number | null;
@@ -93,6 +150,9 @@ export interface HetznerRead {
images: HetznerImage[] | null;
primaryIps: HetznerPrimaryIp[] | null;
prices: HetznerPrices | null;
loadBalancers: HetznerLoadBalancer[] | null;
floatingIps: HetznerFloatingIp[] | null;
certificates: HetznerCertificate[] | null;
}
export interface HetznerAdapter {
@@ -199,6 +259,103 @@ export function parsePrimaryIps(raw: unknown): HetznerPrimaryIp[] {
return out;
}
const HEALTH = new Set(["healthy", "unhealthy", "unknown"]);
/** Pure: the health of one target, from the status of each of its ports. */
function targetHealth(raw: unknown): HetznerLbTarget["health"] {
const states = arr(raw).map((h) => str(obj(h).status)).filter((s): s is string => s !== null && HEALTH.has(s));
if (states.length === 0) return "unknown";
if (states.includes("unhealthy")) return "unhealthy";
return states.every((s) => s === "healthy") ? "healthy" : "unknown";
}
/** Pure: what a load balancer sends to, one entry for each server (a label selector is listed as the servers it found, or as itself when it found none). */
export function parseLbTargets(raw: unknown): HetznerLbTarget[] {
const out: HetznerLbTarget[] = [];
for (const item of arr(raw)) {
const t = obj(item);
if (t.type === "server") {
out.push({ kind: "server", serverId: num(obj(t.server).id), label: null, health: targetHealth(t.health_status) });
} else if (t.type === "ip") {
out.push({ kind: "ip", serverId: null, label: str(obj(t.ip).ip), health: targetHealth(t.health_status) });
} else if (t.type === "label_selector") {
const selector = str(obj(t.label_selector).selector);
const found = arr(t.targets).map(obj).filter((x) => num(obj(x.server).id) !== null);
if (found.length === 0) out.push({ kind: "label_selector", serverId: null, label: selector, health: targetHealth(t.health_status) });
for (const f of found) out.push({ kind: "label_selector", serverId: num(obj(f.server).id), label: selector, health: targetHealth(f.health_status) });
}
}
return out;
}
/** Pure: the load balancers of one page of /load_balancers. */
export function parseLoadBalancers(raw: unknown): HetznerLoadBalancer[] {
const out: HetznerLoadBalancer[] = [];
for (const item of arr(obj(raw).load_balancers)) {
const l = obj(item);
const id = num(l.id);
const name = str(l.name);
if (id === null || !name) continue;
const type = obj(l.load_balancer_type);
const locationName = str(obj(l.location).name);
const priced = arr(type.prices).map(obj).find((p) => str(p.location) === locationName);
const net = obj(l.public_net);
out.push({
id,
name,
type: str(type.name),
location: locationName,
ipv4: str(obj(net.ipv4).ip),
ipv6: str(obj(net.ipv6).ip),
algorithm: str(obj(l.algorithm).type),
services: arr(l.services).length,
targets: parseLbTargets(l.targets),
includedTraffic: num(l.included_traffic),
outgoingTraffic: num(l.outgoing_traffic),
monthlyPrice: price(obj(obj(priced).price_monthly).gross),
});
}
return out;
}
/** Pure: the floating IPs of one page of /floating_ips. */
export function parseFloatingIps(raw: unknown): HetznerFloatingIp[] {
const out: HetznerFloatingIp[] = [];
for (const item of arr(obj(raw).floating_ips)) {
const f = obj(item);
const id = num(f.id);
const ip = str(f.ip);
if (id === null || !ip || (f.type !== "ipv4" && f.type !== "ipv6")) continue;
out.push({ id, name: str(f.name), ip, type: f.type, serverId: num(f.server), location: str(obj(f.home_location).name), description: str(f.description) });
}
return out;
}
/** Pure: the certificates of one page of /certificates. */
export function parseCertificates(raw: unknown): HetznerCertificate[] {
const out: HetznerCertificate[] = [];
for (const item of arr(obj(raw).certificates)) {
const c = obj(item);
const id = num(c.id);
const name = str(c.name);
if (id === null || !name) continue;
const status = obj(c.status);
const after = str(c.not_valid_after);
out.push({
id,
name,
type: str(c.type) ?? "uploaded",
domains: arr(c.domain_names).map((d) => str(d)).filter((d): d is string => d !== null),
notValidAfter: after !== null && Number.isFinite(Date.parse(after)) ? new Date(after).toISOString() : null,
issuance: str(status.issuance),
renewal: str(status.renewal),
error: str(obj(status.error).message),
usedBy: arr(c.used_by).length,
});
}
return out;
}
/** Pure: what /pricing says that the cost estimate needs; null when it is not Hetzner's price list. */
export function parsePrices(raw: unknown): HetznerPrices | null {
const p = obj(obj(raw).pricing);
@@ -214,8 +371,22 @@ export function parsePrices(raw: unknown): HetznerPrices | null {
if (monthly !== null && location && type) primaryIps.push({ type, location, monthly });
}
}
const floatingIps: HetznerPrices["floatingIps"] = [];
for (const kind of arr(p.floating_ips)) {
const k = obj(kind);
for (const entry of arr(k.prices)) {
const monthly = price(obj(obj(entry).price_monthly).gross);
const location = str(obj(entry).location);
const type = str(k.type);
if (monthly !== null && location && type) floatingIps.push({ type, location, monthly });
}
}
// An older price list has one price for a floating IP, whatever its kind and place.
const legacy = price(obj(obj(p.floating_ip).price_monthly).gross);
if (floatingIps.length === 0 && legacy !== null) floatingIps.push({ type: "ipv4", location: "*", monthly: legacy }, { type: "ipv6", location: "*", monthly: legacy });
return {
currency,
floatingIps,
imagePerGb: price(obj(obj(p.image).price_per_gb_month).gross),
volumePerGb: price(obj(obj(p.volume).price_per_gb_month).gross),
backupPercent: price(obj(p.server_backup).percentage),
@@ -258,14 +429,17 @@ export function createHetznerAdapter(config: HetznerConfig): HetznerAdapter {
async function read(): Promise<HetznerRead> {
const servers = await all("/servers", parseServers);
const [volumes, snapshots, backups, primaryIps, prices] = await Promise.all([
const [volumes, snapshots, backups, primaryIps, prices, loadBalancers, floatingIps, certificates] = await Promise.all([
soft(() => all("/volumes", parseVolumes)),
soft(() => all("/images?type=snapshot", parseImages)),
soft(() => all("/images?type=backup", parseImages)),
soft(() => all("/primary_ips", parsePrimaryIps)),
soft(async () => parsePrices(await get("/pricing"))),
soft(() => all("/load_balancers", parseLoadBalancers)),
soft(() => all("/floating_ips", parseFloatingIps)),
soft(() => all("/certificates", parseCertificates)),
]);
return { servers, volumes, images: snapshots && backups ? [...snapshots, ...backups] : null, primaryIps, prices };
return { servers, volumes, images: snapshots && backups ? [...snapshots, ...backups] : null, primaryIps, prices, loadBalancers, floatingIps, certificates };
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
+5
View File
@@ -24,6 +24,7 @@ import { createNetBoxAdapter } from "./netbox/adapter.js";
import { createHetznerAdapter } from "./hetzner/adapter.js";
import { createSpeedtestAdapter } from "./speedtest/adapter.js";
import { createFragnetAdapter } from "./fragnet/adapter.js";
import { createArrAdapter } from "./arr/adapter.js";
import { createNextcloudAdapter } from "./nextcloud/adapter.js";
export interface PingableAdapter {
@@ -85,6 +86,10 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
return createHetznerAdapter(config as any);
case "fragnet":
return createFragnetAdapter(config as any);
case "sonarr":
case "radarr":
case "prowlarr":
return createArrAdapter(type, config as any);
case "netbox":
return createNetBoxAdapter(config as any);
case "nextcloud":
+46
View File
@@ -0,0 +1,46 @@
import { Router } from "express";
import { inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { arrState, integrations } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { ARR_KINDS, type ArrKind } from "../integrations/arr/adapter.js";
import { arrProblems, arrSyncStatuses, distinctDisks, isArrFresh, stuckItems, storedOf } from "../services/arrSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const arrRouter = Router();
arrRouter.use(requireAuth);
/** Every Sonarr, Radarr and Prowlarr, how it is, from what the last reads stored: what it says is wrong, its queue (with what is stuck), disks, missing items and indexers. Everyone signed in can see it. Nothing here changes anything in them. */
arrRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, states, statuses] = await Promise.all([
db.select({ id: integrations.id, type: integrations.type, name: integrations.name, enabled: integrations.enabled, baseUrl: integrations.baseUrl }).from(integrations).where(inArray(integrations.type, ARR_KINDS)).orderBy(integrations.name),
db.select().from(arrState),
arrSyncStatuses(),
]);
const byId = new Map(states.map((s) => [s.integrationId, s]));
const now = Date.now();
res.json({
apps: rows.map((r) => {
const row = byId.get(r.id);
const state = row ? storedOf(row) : null;
const status = statuses[r.id] ?? null;
const fresh = isArrFresh(status ?? undefined, now);
return {
id: r.id,
kind: r.type as ArrKind,
name: r.name,
enabled: r.enabled,
url: r.baseUrl,
status,
fresh,
state: state && {
...state,
disks: state.disks && distinctDisks(state.disks),
/** The ids of the queue items that are stuck, so the page marks them the way the alert counts them. */
stuck: stuckItems(state.queue, now).map((i) => i.id),
},
problems: state && fresh ? arrProblems(r.type as ArrKind, r.id, r.name, state, now) : [],
};
}),
});
}));
+13 -5
View File
@@ -3,7 +3,7 @@ import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { hetznerState, integrations, servers } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { costOf, hetznerProblems, hetznerSyncStatuses, imagesOf, isHetznerFresh, matchAppServer, serversOf, volumesOf } from "../services/hetznerSync.js";
import { certificatesOf, costOf, floatingIpsOf, hetznerProblems, hetznerSyncStatuses, imagesOf, isHetznerFresh, loadBalancersOf, matchAppServer, serversOf, trafficForecast, volumesOf, type StateRow } from "../services/hetznerSync.js";
import { getSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -20,6 +20,9 @@ const parseIps = (stored: string | null): string[] => {
}
};
/** What else of a project the problems look at, from what is stored. */
const extrasOf = (state: StateRow) => ({ loadBalancers: loadBalancersOf(state), floatingIps: floatingIpsOf(state), certificates: certificatesOf(state) });
const appServers = async () => (await db.select({ id: servers.id, name: servers.name, ips: servers.ipAddresses }).from(servers)).map((s) => ({ id: s.id, name: s.name, ips: parseIps(s.ips) }));
/** Every Hetzner project, how it is, from what the last read stored: its servers (and which of them the app also knows by its agent), volumes, snapshots, cost and what is wrong. Everyone signed in can see it. */
@@ -49,7 +52,11 @@ hetznerRouter.get("/", asyncHandler(async (_req, res) => {
fresh,
state: state
? {
servers: stored.map((s) => ({ ...s, linked: matchAppServer(s, known) })),
servers: stored.map(({ trafficHistory: _history, ...s }) => ({ ...s, linked: matchAppServer(s, known), forecast: trafficForecast({ ...s, trafficHistory: _history }, now) })),
/** Null when the load balancers could not be listed. */
loadBalancers: loadBalancersOf(state)?.map((l) => ({ ...l, targets: l.targets.map((t) => ({ ...t, serverName: t.serverId === null ? null : (names.get(t.serverId) ?? null) })) })) ?? null,
floatingIps: floatingIpsOf(state)?.map((f) => ({ ...f, serverName: f.serverId === null ? null : (names.get(f.serverId) ?? null) })) ?? null,
certificates: certificatesOf(state),
/** Null when the volumes could not be listed. */
volumes: volumesOf(state)?.map((v) => ({ ...v, serverName: v.serverId === null ? null : (names.get(v.serverId) ?? null) })) ?? null,
images: images?.map((i) => ({ ...i, serverName: i.serverId === null ? null : (names.get(i.serverId) ?? null) })) ?? null,
@@ -57,7 +64,7 @@ hetznerRouter.get("/", asyncHandler(async (_req, res) => {
updatedAt: state.updatedAt,
}
: null,
problems: state && fresh ? hetznerProblems(r.id, r.name, stored, images, now, settings.healthChecks.backupStaleHours) : [],
problems: state && fresh ? hetznerProblems(r.id, r.name, stored, images, now, settings.healthChecks.backupStaleHours, extrasOf(state)) : [],
};
}),
});
@@ -91,10 +98,11 @@ hetznerRouter.get("/for-server/:id", asyncHandler(async (req, res) => {
integrationName: r.name,
fresh,
updatedAt: state.updatedAt,
server,
server: (({ trafficHistory: _h, ...rest }) => rest)(server),
newestBackup,
snapshots: (images ?? []).filter((i) => i.type === "snapshot" && i.serverId === server.id).length,
problems: fresh ? hetznerProblems(r.id, r.name, stored, images, now, settings.healthChecks.backupStaleHours).filter((p) => p.serverId === server.id) : [],
problems: fresh ? hetznerProblems(r.id, r.name, stored, images, now, settings.healthChecks.backupStaleHours, extrasOf(state)).filter((p) => p.serverId === server.id) : [],
forecast: trafficForecast(server, now),
},
});
}
+4
View File
@@ -39,6 +39,7 @@ import { forgetNetBox, syncNetBox } from "../services/netboxSync.js";
import { forgetHetzner, syncHetzner } from "../services/hetznerSync.js";
import { forgetSpeedtest, syncSpeedtest } from "../services/speedtestSync.js";
import { forgetFragnet, syncFragnet } from "../services/fragnetSync.js";
import { forgetArr, syncArr } from "../services/arrSync.js";
import { forgetNextcloud, syncNextcloud } from "../services/nextcloudSync.js";
import { dockerVulnSyncStatuses, findingsOf, forgetDockerVulns, isDockerVulnFresh, syncDockerVulns } from "../services/dockerVulnSync.js";
import { invalidateAlertsCache } from "../services/alerts.js";
@@ -153,6 +154,7 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
if (type === "hetzner") void syncHetzner(created.id).then(() => invalidateAlertsCache());
if (type === "speedtest") void syncSpeedtest(created.id).then(() => invalidateAlertsCache());
if (type === "fragnet") void syncFragnet(created.id).then(() => invalidateAlertsCache());
if (type === "sonarr" || type === "radarr" || type === "prowlarr") void syncArr(created.id).then(() => invalidateAlertsCache());
if (type === "nextcloud") void syncNextcloud(created.id).then(() => invalidateAlertsCache());
if (type === "dockhand") void syncDockerVulns(created.id).then(() => invalidateAlertsCache());
@@ -268,6 +270,7 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
if (updated.type === "hetzner" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncHetzner(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "speedtest" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncSpeedtest(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "fragnet" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncFragnet(updated.id).then(() => invalidateAlertsCache());
if ((updated.type === "sonarr" || updated.type === "radarr" || updated.type === "prowlarr") && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncArr(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "nextcloud" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncNextcloud(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "dockhand" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncDockerVulns(updated.id).then(() => invalidateAlertsCache());
@@ -359,6 +362,7 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
if (existing.type === "hetzner") await forgetHetzner(id);
if (existing.type === "speedtest") await forgetSpeedtest(id);
if (existing.type === "fragnet") await forgetFragnet(id);
if (existing.type === "sonarr" || existing.type === "radarr" || existing.type === "prowlarr") await forgetArr(id);
if (existing.type === "nextcloud") await forgetNextcloud(id);
if (existing.type === "dockhand") await forgetDockerVulns(id);
await db.delete(integrations).where(eq(integrations.id, id));
+9 -4
View File
@@ -86,6 +86,9 @@ const SOURCES: Record<string, { label: string; link: string }> = {
hetzner: { label: "Hetzner", link: "/hetzner" },
speedtest: { label: "Speedtest", link: "/speedtest" },
fragnet: { label: "Fragnet", link: "/game-servers" },
sonarr: { label: "Sonarr", link: "/media" },
radarr: { label: "Radarr", link: "/media" },
prowlarr: { label: "Prowlarr", link: "/media" },
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/dns?view=domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
@@ -126,8 +129,10 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
if (m) return { type: "speedtest", id: Number(m[1]), category: "monitoring" };
m = /^hetzner-(off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
m = /^hetzner-(off|stuck|rescue|traffic|forecast|backup|lb|cert|fip):(\d+):/.exec(key);
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" || m[1] === "forecast" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
m = /^(sonarr|radarr|prowlarr)-(health|queue|disk):(\d+)/.exec(key);
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
if (m) return { type: "fragnet", id: Number(m[2]), category: m[1] === "disk" ? "disk" : m[1] === "backup" ? "backup" : "monitoring" };
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
@@ -270,7 +275,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : undefined,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : origin.type === "sonarr" || origin.type === "radarr" || origin.type === "prowlarr" ? "/media" : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
@@ -280,7 +285,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet):(\d+)(?::(.+))?$/.exec(h);
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
+227
View File
@@ -0,0 +1,227 @@
/**
* Keeps what Sonarr, Radarr and Prowlarr say up to date: every five minutes each one is read — what its own checks found wrong, its download queue, its disks, how many wanted items are
* missing and (Prowlarr) its indexers — and the state is stored. Pages and alerts are built from what is stored, so none of them talks to the applications.
*
* What is raised: **what the application itself says is wrong** (its health checks of type warning or error: a download client that cannot be reached, indexers switched off after failures, a
* root folder gone — a mere notice, such as an update being available, is not raised); **downloads that are stuck** — in the queue for two hours or more and judged warning or error, or failed,
* or blocked from importing (one alert for the lot, naming the first few); **a disk nearly full** (under 10% free, critical under 5%). The same three for each of the three, under one key each.
*/
import { and, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { arrState, integrations } from "../db/schema.js";
import { ARR_KINDS, ARR_NAMES, createArrAdapter, type ArrDisk, type ArrHealthIssue, type ArrIndexer, type ArrKind, type ArrQueueItem, type ArrRead } from "../integrations/arr/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 5 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 20 * 60_000;
const STATUS_FLAG = "arrSync";
const MIN = 60_000;
const HOUR = 60 * MIN;
/** A download judged wrong for less than this is left to sort itself out (a client that is just slow to answer). */
export const STUCK_AFTER_MS = 2 * HOUR;
export const DISK_WARN_FREE = 0.1;
export const DISK_CRITICAL_FREE = 0.05;
/** A disk smaller than this is not a media disk (a tmpfs, a boot partition). */
export const MIN_DISK_BYTES = 2 * 1024 ** 3;
export interface ArrSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, ArrSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, ArrSyncStatus>) : {};
} catch {
return {};
}
}
export async function arrSyncStatuses(): Promise<Record<number, ArrSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isArrFresh = (status: ArrSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof arrState.$inferSelect;
function parse<T>(stored: string | null): T | null {
if (stored === null) return null;
try {
return JSON.parse(stored) as T;
} catch {
return null;
}
}
/** What is stored of one application, as the read gave it. */
export interface StoredArr {
version: string | null;
health: ArrHealthIssue[];
queue: { total: number; items: ArrQueueItem[] } | null;
disks: ArrDisk[] | null;
missing: number | null;
indexers: ArrIndexer[] | null;
updatedAt: string;
}
export const storedOf = (row: StateRow): StoredArr => ({
version: row.version,
health: parse<ArrHealthIssue[]>(row.health) ?? [],
queue: parse<{ total: number; items: ArrQueueItem[] }>(row.queue),
disks: parse<ArrDisk[]>(row.disks),
missing: row.missing,
indexers: parse<ArrIndexer[]>(row.indexers),
updatedAt: row.updatedAt,
});
/** Writes down one read, replacing the last. */
export async function storeRead(integrationId: number, r: ArrRead, now: Date = new Date()): Promise<void> {
const values = {
version: r.version,
health: JSON.stringify(r.health),
queue: r.queue && JSON.stringify(r.queue),
disks: r.disks && JSON.stringify(r.disks),
missing: r.missing,
indexers: r.indexers && JSON.stringify(r.indexers),
updatedAt: now.toISOString(),
};
await db.insert(arrState).values({ integrationId, ...values }).onConflictDoUpdate({ target: arrState.integrationId, set: values });
}
/** Reads one application and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncArr(integrationId: number, now: Date = new Date(), read?: (kind: ArrKind, config: Record<string, unknown>) => Promise<ArrRead>): Promise<ArrSyncStatus> {
let status: ArrSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
const kind = loaded?.integration.type as ArrKind | undefined;
if (!loaded || !kind || !ARR_KINDS.includes(kind)) throw new Error("not a Sonarr, Radarr or Prowlarr integration");
const result = await (read ? read(kind, loaded.config) : createArrAdapter(kind, loaded.config as never).read());
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[arr] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[arr] couldn't record the result:", err);
}
return status;
}
export async function syncAllArr(now: Date = new Date()): Promise<Record<number, ArrSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(inArray(integrations.type, ARR_KINDS), eq(integrations.enabled, true)));
const out: Record<number, ArrSyncStatus> = {};
for (const r of rows) out[r.id] = await syncArr(r.id, now);
return out;
}
export async function forgetArr(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface ArrProblem {
key: string;
message: string;
severity: "warning" | "critical";
}
const slug = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40);
/** A short stable stamp of a message, numbers left out (so "3 indexers" and "4 indexers" are the same finding). */
function stamp(message: string): string {
let h = 0;
for (const c of message.replace(/\d+/g, "#")) h = (h * 31 + c.charCodeAt(0)) >>> 0;
return h.toString(36);
}
export const sizeText = (bytes: number): string => {
const gb = bytes / 1024 ** 3;
return gb >= 1024 ? `${(gb / 1024).toFixed(1)} TB` : `${gb.toFixed(gb >= 100 ? 0 : 1)} GB`;
};
/** A finding starts with a capital as a sentence of its own; after "says" it reads better without ("No download client" → "no download client"), unless it starts with a name or an acronym. */
const lcFirst = (s: string) => (/^[A-Z][a-z]/.test(s) ? s[0].toLowerCase() + s.slice(1) : s);
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / (24 * HOUR)), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
/** Pure: whether a queue item is stuck — judged wrong, or failed, or blocked from importing — and has been in the queue long enough for it not to sort itself out. */
export function isStuck(i: ArrQueueItem, nowMs: number): boolean {
const bad = i.tracked === "warning" || i.tracked === "error" || i.state === "failed" || i.state === "failedPending" || i.state === "importBlocked" || i.status === "failed";
if (!bad) return false;
const since = i.added === null ? null : nowMs - Date.parse(i.added);
return since !== null && since >= STUCK_AFTER_MS;
}
/** Pure: the stuck items, longest in the queue first. */
export const stuckItems = (queue: StoredArr["queue"], nowMs: number): ArrQueueItem[] => (queue?.items ?? []).filter((i) => isStuck(i, nowMs)).sort((a, b) => Date.parse(a.added ?? "") - Date.parse(b.added ?? ""));
/** Pure: the disks worth judging — one for each filesystem (the same size and free space is the same disk, however many folders are on it). */
export function distinctDisks(disks: ArrDisk[] | null): ArrDisk[] {
const seen = new Set<string>();
const out: ArrDisk[] = [];
for (const d of [...(disks ?? [])].sort((a, b) => a.path.length - b.path.length || a.path.localeCompare(b.path))) {
if (d.total < MIN_DISK_BYTES) continue;
const id = `${d.total}:${d.free}`;
if (seen.has(id)) continue;
seen.add(id);
out.push(d);
}
return out;
}
/** Pure: what is wrong with one application, from what is stored for it. */
export function arrProblems(kind: ArrKind, id: number, name: string, state: StoredArr, nowMs: number): ArrProblem[] {
const out: ArrProblem[] = [];
const app = ARR_NAMES[kind];
for (const h of state.health) {
if (h.type !== "warning" && h.type !== "error") continue;
out.push({ key: `${kind}-health:${id}:${slug(h.source)}-${stamp(h.message)}`, severity: "warning", message: `${name}: ${app} says ${lcFirst(h.message.trim())}` });
}
const stuck = stuckItems(state.queue, nowMs);
if (stuck.length > 0) {
const names = stuck.slice(0, 3).map((i) => i.title);
const oldest = stuck[0].added === null ? null : nowMs - Date.parse(stuck[0].added);
const why = stuck[0].messages[0];
out.push({
key: `${kind}-queue:${id}`,
severity: "warning",
message: `${name}: ${plural(stuck.length, "download")} ${stuck.length === 1 ? "is" : "are"} stuck — ${names.join("; ")}${stuck.length > 3 ? `; and ${stuck.length - 3} more` : ""}${oldest !== null ? ` (the oldest for ${ago(oldest)})` : ""}${why ? ` — ${why}` : ""}`,
});
}
for (const d of distinctDisks(state.disks)) {
const free = d.free / d.total;
if (free < DISK_WARN_FREE) {
out.push({ key: `${kind}-disk:${id}:${slug(d.path)}`, severity: free < DISK_CRITICAL_FREE ? "critical" : "warning", message: `${name}: ${d.path} has ${sizeText(d.free)} free of ${sizeText(d.total)} (${Math.floor(free * 100)}%)` });
}
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initArrSync(): void {
if (timer) return;
const pass = () => void syncAllArr().catch((err) => console.error("[arr] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 37_000);
console.log(`[arr] reading Sonarr, Radarr and Prowlarr every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+29 -4
View File
@@ -1,6 +1,6 @@
import { and, eq } from "drizzle-orm";
import { and, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState } from "../db/schema.js";
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
@@ -30,7 +30,9 @@ import { isNetBoxFresh, netboxProblems, netboxSyncStatuses, type NetBoxProblem }
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type NextcloudProblem } from "./nextcloudSync.js";
import { dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, type DockerVulnProblem } from "./dockerVulnSync.js";
import { expectedOf as speedtestExpected, isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestProblems, speedtestSyncStatuses, type SpeedtestProblem } from "./speedtestSync.js";
import { hetznerProblems, hetznerSyncStatuses, imagesOf as hetznerImagesOf, isHetznerFresh, serversOf as hetznerServersOf, type HetznerProblem } from "./hetznerSync.js";
import { hetznerProblems, hetznerSyncStatuses, imagesOf as hetznerImagesOf, loadBalancersOf as hetznerLoadBalancersOf, floatingIpsOf as hetznerFloatingIpsOf, certificatesOf as hetznerCertificatesOf, isHetznerFresh, serversOf as hetznerServersOf, type HetznerProblem } from "./hetznerSync.js";
import { ARR_KINDS, type ArrKind } from "../integrations/arr/adapter.js";
import { arrProblems, arrSyncStatuses, isArrFresh, storedOf as arrStoredOf, type ArrProblem } from "./arrSync.js";
import { fragnetProblems, fragnetSyncStatuses, isFragnetFresh, optionsOf as fragnetOptions, serversOf as fragnetServersOf, type FragnetProblem } from "./fragnetSync.js";
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
import { recordStorageSamples } from "./storageHistory.js";
@@ -128,6 +130,8 @@ export interface HealthSnapshot {
speedtest?: { integrationId: number; problems: SpeedtestProblem[] }[];
/** What is wrong with each Fragnet game panel account (a backup that is late, a disk or memory nearly full), only while that reading is recent (services/fragnetSync.ts). */
fragnet?: { integrationId: number; problems: FragnetProblem[] }[];
/** What is wrong with each Sonarr, Radarr and Prowlarr (what it says itself, downloads that are stuck, a disk nearly full), only while that reading is recent (services/arrSync.ts). */
arr?: { integrationId: number; kind: ArrKind; problems: ArrProblem[] }[];
/** Containers to leave alone. */
dockerIgnore?: string[];
/** The clock the container judgements are made on (the pass's, so a test can set it). */
@@ -269,6 +273,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
// Speedtest Tracker: the internet slower than it is sold at, speed tests that keep failing.
for (const s of snapshot.speedtest ?? []) for (const p of s.problems) out.push({ key: p.key, source: `speedtest:${s.integrationId}`, message: p.message, severity: p.severity });
// Sonarr, Radarr and Prowlarr: what they say is wrong, downloads that are stuck, disks nearly full.
for (const a of snapshot.arr ?? []) for (const p of a.problems) out.push({ key: p.key, source: `${a.kind}:${a.integrationId}`, message: p.message, severity: p.severity });
// Fragnet game servers: backups that are late, disks and memory nearly full.
for (const f of snapshot.fragnet ?? []) for (const p of f.problems) out.push({ key: p.key, source: `fragnet:${f.integrationId}`, message: p.message, severity: p.severity });
@@ -537,6 +544,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
console.error("[health] couldn't judge Speedtest Tracker:", err instanceof Error ? err.message : err);
}
// Sonarr, Radarr and Prowlarr: read by their own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsA = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(and(inArray(integrations.type, ARR_KINDS), eq(integrations.enabled, true)));
if (rowsA.length > 0) {
const statuses = await arrSyncStatuses();
for (const r of rowsA) {
const [state] = await db.select().from(arrState).where(eq(arrState.integrationId, r.id)).limit(1);
if (!state || !isArrFresh(statuses[r.id], Date.now())) {
held.add(`${r.type}:${r.id}`);
continue;
}
(snapshot.arr ??= []).push({ integrationId: r.id, kind: r.type as ArrKind, problems: arrProblems(r.type as ArrKind, r.id, r.name, arrStoredOf(state), Date.now()) });
}
}
} catch (err) {
console.error("[health] couldn't judge Sonarr, Radarr and Prowlarr:", err instanceof Error ? err.message : err);
}
// Fragnet game servers: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsF = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "fragnet"), eq(integrations.enabled, true)));
@@ -567,7 +592,7 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
held.add(`hetzner:${r.id}`);
continue;
}
(snapshot.hetzner ??= []).push({ integrationId: r.id, problems: hetznerProblems(r.id, r.name, hetznerServersOf(state), hetznerImagesOf(state), Date.now(), checks.backupStaleHours) });
(snapshot.hetzner ??= []).push({ integrationId: r.id, problems: hetznerProblems(r.id, r.name, hetznerServersOf(state), hetznerImagesOf(state), Date.now(), checks.backupStaleHours, { loadBalancers: hetznerLoadBalancersOf(state), floatingIps: hetznerFloatingIpsOf(state), certificates: hetznerCertificatesOf(state) }) });
}
}
} catch (err) {
+143 -5
View File
@@ -10,7 +10,7 @@
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { hetznerState, integrations } from "../db/schema.js";
import { createHetznerAdapter, type HetznerImage, type HetznerRead, type HetznerServer, type HetznerVolume } from "../integrations/hetzner/adapter.js";
import { createHetznerAdapter, type HetznerCertificate, type HetznerFloatingIp, type HetznerImage, type HetznerLoadBalancer, type HetznerRead, type HetznerServer, type HetznerVolume } from "../integrations/hetzner/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
@@ -28,6 +28,17 @@ export const STUCK_AFTER_MS = 30 * MIN;
/** Rescue mode switched on and not used within this long: the next reboot would come up in it. */
export const RESCUE_AFTER_MS = HOUR;
export const TRAFFIC_WARN = 0.8;
const DAY = 24 * HOUR;
/** A certificate this close to its end is a warning; this close (or past it) is critical. */
export const CERT_WARN_DAYS = 14;
export const CERT_CRITICAL_DAYS = 3;
/** The traffic of a server is written down about this often (more reads than that are not kept), for the last two weeks. */
export const TRAFFIC_SAMPLE_EVERY_MS = 6 * HOUR;
export const TRAFFIC_SAMPLE_KEEP_MS = 14 * DAY;
/** A forecast needs this much of the month behind it (by the month's average), or this much of the last week (by the recent rate). */
export const FORECAST_MIN_MONTH_MS = 3 * DAY;
export const FORECAST_MIN_RECENT_MS = 2 * DAY;
export const FORECAST_RECENT_WINDOW_MS = 7 * DAY;
export interface HetznerSyncStatus {
/** The last read that worked (kept when a later one failed). */
@@ -61,6 +72,13 @@ export interface StoredServer extends HetznerServer {
statusSince: string;
/** When the first read that showed rescue mode on was made; null when it is not on. */
rescueSince: string | null;
/** The month's outgoing traffic as it was now and then (about every six hours, for two weeks, starting over when the counter goes down — a new month); what the forecast is made from. */
trafficHistory?: TrafficSample[];
}
export interface TrafficSample {
at: string;
bytes: number;
}
export interface HetznerCost {
@@ -70,6 +88,9 @@ export interface HetznerCost {
volumes: number;
snapshots: number;
ips: number;
/** Load balancers and floating IPs (absent in an estimate stored before they were read). */
loadBalancers?: number;
floatingIps?: number;
total: number;
/** Primary IPs that belong to no server (billed all the same). */
unassignedIps: number;
@@ -90,6 +111,9 @@ const parseList = <T>(stored: string | null): T[] | null => {
export const serversOf = (row: Pick<StateRow, "servers">): StoredServer[] => parseList<StoredServer>(row.servers) ?? [];
export const volumesOf = (row: Pick<StateRow, "volumes">): HetznerVolume[] | null => parseList<HetznerVolume>(row.volumes);
export const imagesOf = (row: Pick<StateRow, "images">): HetznerImage[] | null => parseList<HetznerImage>(row.images);
export const loadBalancersOf = (row: Pick<StateRow, "loadBalancers">): HetznerLoadBalancer[] | null => parseList<HetznerLoadBalancer>(row.loadBalancers);
export const floatingIpsOf = (row: Pick<StateRow, "floatingIps">): HetznerFloatingIp[] | null => parseList<HetznerFloatingIp>(row.floatingIps);
export const certificatesOf = (row: Pick<StateRow, "certificates">): HetznerCertificate[] | null => parseList<HetznerCertificate>(row.certificates);
export const costOf = (row: Pick<StateRow, "cost">): HetznerCost | null => {
if (!row.cost) return null;
try {
@@ -99,6 +123,55 @@ export const costOf = (row: Pick<StateRow, "cost">): HetznerCost | null => {
}
};
/** Pure: the traffic history with this read added (one sample about every six hours); a counter that went down is a new month, so the history starts over; old samples are dropped. */
export function carryTraffic(bytes: number | null, history: TrafficSample[], now: Date): TrafficSample[] {
if (bytes === null) return history;
const nowMs = now.getTime();
let kept = history.filter((h) => nowMs - Date.parse(h.at) <= TRAFFIC_SAMPLE_KEEP_MS);
const last = kept[kept.length - 1];
if (last && bytes < last.bytes) kept = [];
const newest = kept[kept.length - 1];
if (!newest || nowMs - Date.parse(newest.at) >= TRAFFIC_SAMPLE_EVERY_MS) kept = [...kept, { at: now.toISOString(), bytes }];
return kept;
}
export interface TrafficForecast {
/** Bytes a day, the way it has been going. */
perDay: number;
/** What the month will have come to by its end at that rate. */
projectedMonth: number;
/** When the quota is passed, if it is before the month ends; null when it is not (or already has been). */
exhaustedAt: string | null;
/** Whether the rate is the last week's or the month's average. */
basis: "recent" | "month";
}
/**
* Pure: how the month's traffic is going to end, by the rate of the last week (or, with too little of it, the month's average so far) — null when there is too little to say anything. The
* month is taken to be the calendar month in UTC, which is how Hetzner counts it (not checked against a live project: a counter that goes down starts the history over anyway).
*/
export function trafficForecast(s: Pick<StoredServer, "outgoingTraffic" | "includedTraffic" | "trafficHistory">, nowMs: number): TrafficForecast | null {
const used = s.outgoingTraffic;
const included = s.includedTraffic;
if (used === null || included === null || included <= 0 || used <= 0) return null;
const d = new Date(nowMs);
const monthStart = Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), 1);
const monthEnd = Date.UTC(d.getUTCFullYear(), d.getUTCMonth() + 1, 1);
let perDay: number | null = null;
let basis: TrafficForecast["basis"] = "recent";
const window = (s.trafficHistory ?? []).filter((h) => Date.parse(h.at) >= Math.max(monthStart, nowMs - FORECAST_RECENT_WINDOW_MS) && h.bytes <= used);
const oldest = window[0];
if (oldest && nowMs - Date.parse(oldest.at) >= FORECAST_MIN_RECENT_MS) perDay = ((used - oldest.bytes) / (nowMs - Date.parse(oldest.at))) * DAY;
if (perDay === null && nowMs - monthStart >= FORECAST_MIN_MONTH_MS) {
perDay = (used / (nowMs - monthStart)) * DAY;
basis = "month";
}
if (perDay === null) return null;
const projectedMonth = used + (perDay * (monthEnd - nowMs)) / DAY;
const exhaustedAt = used < included && projectedMonth > included && perDay > 0 ? new Date(nowMs + ((included - used) / perDay) * DAY).toISOString() : null;
return { perDay, projectedMonth, exhaustedAt, basis };
}
/** Pure: what the app has seen between reads, carried from the last read of the same server. A server first seen starts its clocks now. */
export function carryTimes(servers: HetznerServer[], previous: StoredServer[], now: Date): StoredServer[] {
const before = new Map(previous.map((p) => [p.id, p]));
@@ -107,6 +180,7 @@ export function carryTimes(servers: HetznerServer[], previous: StoredServer[], n
const p = before.get(s.id);
return {
...s,
trafficHistory: carryTraffic(s.outgoingTraffic, p?.trafficHistory ?? [], now),
lastRunningAt: s.status === "running" ? at : (p?.lastRunningAt ?? null),
statusSince: p && p.status === s.status ? p.statusSince : at,
rescueSince: s.rescueEnabled ? (p?.rescueEnabled ? (p.rescueSince ?? at) : at) : null,
@@ -115,7 +189,7 @@ export function carryTimes(servers: HetznerServer[], previous: StoredServer[], n
}
/** Pure: roughly what the project costs per month at Hetzner's list prices, including VAT; null when the price list could not be read. Load balancers, floating IPs and traffic over the quota are not in it. */
export function estimateCost(read: Pick<HetznerRead, "servers" | "volumes" | "images" | "primaryIps" | "prices">): HetznerCost | null {
export function estimateCost(read: Pick<HetznerRead, "servers" | "volumes" | "images" | "primaryIps" | "prices"> & Partial<Pick<HetznerRead, "loadBalancers" | "floatingIps">>): HetznerCost | null {
const p = read.prices;
if (!p) return null;
let servers = 0;
@@ -137,9 +211,21 @@ export function estimateCost(read: Pick<HetznerRead, "servers" | "volumes" | "im
ips += p.primaryIps.find((x) => x.type === ip.type && x.location === ip.location)?.monthly ?? 0;
if (!ip.assigned) unassignedIps++;
}
let loadBalancers = 0;
for (const lb of read.loadBalancers ?? []) {
if (lb.monthlyPrice === null) unpriced++;
else loadBalancers += lb.monthlyPrice;
}
let floatingIps = 0;
for (const f of read.floatingIps ?? []) {
const prices = p.floatingIps ?? [];
const found = prices.find((x) => x.type === f.type && x.location === f.location) ?? prices.find((x) => x.type === f.type && x.location === "*");
if (found) floatingIps += found.monthly;
else unpriced++;
}
const round = (n: number) => Math.round(n * 100) / 100;
const parts = { servers: round(servers), backups: round(backups), volumes: round(volumes), snapshots: round(snapshots), ips: round(ips) };
return { currency: p.currency, ...parts, total: round(servers + backups + volumes + snapshots + ips), unassignedIps, unpriced };
const parts = { servers: round(servers), backups: round(backups), volumes: round(volumes), snapshots: round(snapshots), ips: round(ips), loadBalancers: round(loadBalancers), floatingIps: round(floatingIps) };
return { currency: p.currency, ...parts, total: round(servers + backups + volumes + snapshots + ips + loadBalancers + floatingIps), unassignedIps, unpriced };
}
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
@@ -149,6 +235,9 @@ export async function storeRead(integrationId: number, r: HetznerRead, now: Date
servers: JSON.stringify(carryTimes(r.servers, previous ? serversOf(previous) : [], now)),
volumes: r.volumes && JSON.stringify(r.volumes),
images: r.images && JSON.stringify(r.images),
loadBalancers: r.loadBalancers && JSON.stringify(r.loadBalancers),
floatingIps: r.floatingIps && JSON.stringify(r.floatingIps),
certificates: r.certificates && JSON.stringify(r.certificates),
cost: ((c) => c && JSON.stringify(c))(estimateCost(r)),
updatedAt: now.toISOString(),
};
@@ -204,6 +293,20 @@ export interface HetznerProblem {
severity: "warning" | "critical";
}
/** What else of a project the problems look at: each is null when it could not be read, which says nothing is wrong. */
export interface HetznerExtras {
loadBalancers?: HetznerLoadBalancer[] | null;
floatingIps?: HetznerFloatingIp[] | null;
certificates?: HetznerCertificate[] | null;
}
const MONTHS = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"];
/** "24 June", in UTC (Hetzner's month is). */
export const dayText = (iso: string): string => {
const d = new Date(iso);
return `${d.getUTCDate()} ${MONTHS[d.getUTCMonth()]}`;
};
/** In the units the pages show (binary, written GB or TB), so an alert and the tile beside it agree. */
export const sizeText = (bytes: number): string => {
const gb = bytes / 1024 ** 3;
@@ -216,7 +319,7 @@ const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / (2
/**
* Pure: what is wrong with one project, from what is stored for it. `backupStaleHours` is the same limit as for Proxmox guests' backups (0 = not judged).
*/
export function hetznerProblems(id: number, name: string, servers: StoredServer[], images: HetznerImage[] | null, nowMs: number, backupStaleHours: number): HetznerProblem[] {
export function hetznerProblems(id: number, name: string, servers: StoredServer[], images: HetznerImage[] | null, nowMs: number, backupStaleHours: number, extras: HetznerExtras = {}): HetznerProblem[] {
const out: HetznerProblem[] = [];
for (const s of [...servers].sort((a, b) => a.name.localeCompare(b.name))) {
const since = nowMs - Date.parse(s.statusSince);
@@ -230,6 +333,15 @@ export function hetznerProblems(id: number, name: string, servers: StoredServer[
}
if (s.includedTraffic !== null && s.includedTraffic > 0 && s.outgoingTraffic !== null) {
const used = s.outgoingTraffic / s.includedTraffic;
const forecast = used < TRAFFIC_WARN ? trafficForecast(s, nowMs) : null;
if (forecast?.exhaustedAt) {
out.push({
key: `hetzner-forecast:${id}:${s.id}`,
serverId: s.id,
severity: "warning",
message: `${name}: at this rate ${s.name} will pass its ${sizeText(s.includedTraffic)} traffic quota on ${dayText(forecast.exhaustedAt)} — it has sent ${sizeText(s.outgoingTraffic)} so far, about ${sizeText(forecast.perDay)} a day (${forecast.basis === "recent" ? "the last week's rate" : "the month's average"})`,
});
}
if (used >= TRAFFIC_WARN) {
out.push({
key: `hetzner-traffic:${id}:${s.id}`,
@@ -249,6 +361,32 @@ export function hetznerProblems(id: number, name: string, servers: StoredServer[
}
}
}
const serverName = (sid: number | null) => (sid === null ? null : (servers.find((x) => x.id === sid)?.name ?? `#${sid}`));
for (const lb of [...(extras.loadBalancers ?? [])].sort((a, b) => a.name.localeCompare(b.name))) {
const unhealthy = lb.targets.filter((t) => t.health === "unhealthy");
if (unhealthy.length === 0) continue;
const names = [...new Set(unhealthy.map((t) => serverName(t.serverId) ?? t.label ?? "a target"))];
out.push({
key: `hetzner-lb:${id}:${lb.id}`,
serverId: 0,
severity: unhealthy.length === lb.targets.length ? "critical" : "warning",
message: `${name}: load balancer ${lb.name} has ${unhealthy.length} of ${plural(lb.targets.length, "target")} unhealthy (${names.join(", ")})${unhealthy.length === lb.targets.length ? " — it has nothing healthy to send to" : ""}`,
});
}
for (const c of [...(extras.certificates ?? [])].sort((a, b) => a.name.localeCompare(b.name))) {
if (c.usedBy === 0) continue; // one nothing uses ending is no news
const label = `certificate ${c.name}${c.domains.length > 0 ? ` (${c.domains.slice(0, 3).join(", ")}${c.domains.length > 3 ? ", …" : ""})` : ""}`;
const left = c.notValidAfter === null ? null : (Date.parse(c.notValidAfter) - nowMs) / DAY;
if (c.issuance === "failed" || c.renewal === "failed") {
out.push({ key: `hetzner-cert:${id}:${c.id}`, serverId: 0, severity: left !== null && left <= CERT_CRITICAL_DAYS ? "critical" : "warning", message: `${name}: ${label} could not be ${c.issuance === "failed" ? "issued" : "renewed"} by Hetzner${c.error ? ` — ${c.error}` : ""}${left !== null && left > 0 ? ` (it is valid for ${plural(Math.floor(left), "more day")})` : ""}` });
} else if (left !== null && left <= CERT_WARN_DAYS) {
out.push({ key: `hetzner-cert:${id}:${c.id}`, serverId: 0, severity: left <= CERT_CRITICAL_DAYS ? "critical" : "warning", message: `${name}: ${label} ${left <= 0 ? `ended ${ago(-left * DAY)} ago` : `ends in ${plural(Math.ceil(left), "day")}`}${c.type === "managed" ? " — Hetzner should have renewed it" : " — it was uploaded, so it has to be renewed by hand"}` });
}
}
for (const f of [...(extras.floatingIps ?? [])].sort((a, b) => a.ip.localeCompare(b.ip))) {
if (f.serverId !== null) continue;
out.push({ key: `hetzner-fip:${id}:${f.id}`, serverId: 0, severity: "warning", message: `${name}: floating IP ${f.ip}${f.name ? ` (${f.name})` : ""} points at no server — it is billed all the same` });
}
return out;
}
+3 -1
View File
@@ -39,10 +39,12 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `integration:${m[1]}`;
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^hetzner-(?:off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
m = /^hetzner-(?:off|stuck|rescue|traffic|forecast|backup|lb|cert|fip):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^fragnet-(?:disk|memory|backup):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^(?:sonarr|radarr|prowlarr)-(?:health|queue|disk):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^netbox-(?:workers):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^n8n-(?:failed):(\d+)/.exec(key);
+35 -3
View File
@@ -3,9 +3,9 @@
* severity, integrations, maintenance, things that expire. It reads what the app already knows — nothing here calls out to a server or an
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
*/
import { eq } from "drizzle-orm";
import { eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState } from "../db/schema.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { backupState, listAll as listDuplicati } from "./duplicati.js";
@@ -13,7 +13,9 @@ import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync
import { isNextcloudFresh, nextcloudSyncStatuses } from "./nextcloudSync.js";
import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf } from "./hetznerSync.js";
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf, loadBalancersOf as hetznerLoadBalancersOf, certificatesOf as hetznerCertificatesOf, trafficForecast as hetznerTrafficForecast } from "./hetznerSync.js";
import { ARR_KINDS } from "../integrations/arr/adapter.js";
import { arrSyncStatuses, distinctDisks as arrDisks, isArrFresh, stuckItems as arrStuck, storedOf as arrStoredOf } from "./arrSync.js";
import { fragnetSyncStatuses, isFragnetFresh, newestBackup as fragnetNewestBackup, serversOf as fragnetServersOf } from "./fragnetSync.js";
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
import { isN8nFresh, n8nSyncStatuses, workflowsOf as n8nWorkflowsOf } from "./n8nSync.js";
@@ -101,6 +103,25 @@ async function addSpeedtestMetrics(out: Exposition, now: Date) {
}
}
/** Each Sonarr, Radarr and Prowlarr, for /metrics: how many health findings it has, its queue and how much of it is stuck, the missing items and the free space of each disk. Only while the reading is recent. */
async function addArrMetrics(out: Exposition, now: Date) {
const statuses = await arrSyncStatuses();
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations).where(inArray(integrations.type, ARR_KINDS))).map((r) => [r.id, r]));
for (const s of await db.select().from(arrState)) {
const row = rows.get(s.integrationId);
if (!row || !row.enabled || !isArrFresh(statuses[s.integrationId], now.getTime())) continue;
const state = arrStoredOf(s);
const labels = { app: row.type, name: row.name };
out.add("homelab_arr_health_issues", "gauge", "How many warnings or errors Sonarr, Radarr or Prowlarr reports about itself.", state.health.filter((h) => h.type === "warning" || h.type === "error").length, labels);
if (state.queue) {
out.add("homelab_arr_queue_items", "gauge", "How many downloads are in the queue of Sonarr or Radarr.", state.queue.total, labels);
out.add("homelab_arr_queue_stuck_items", "gauge", "How many of the queue's downloads are stuck (judged wrong, failed or blocked for two hours or more).", arrStuck(state.queue, now.getTime()).length, labels);
}
if (state.missing !== null) out.add("homelab_arr_wanted_missing", "gauge", "How many wanted items Sonarr or Radarr has no file for.", state.missing, labels);
for (const d of arrDisks(state.disks)) out.add("homelab_arr_disk_free_bytes", "gauge", "Free space of a disk Sonarr, Radarr or Prowlarr uses, in bytes.", d.free, { ...labels, path: d.path });
}
}
/** Each Fragnet game server, for /metrics: whether it is running, its disk and memory against their limits and when it was last backed up. Only while the reading is recent. */
async function addFragnetMetrics(out: Exposition, now: Date) {
const statuses = await fragnetSyncStatuses();
@@ -136,6 +157,16 @@ async function addHetznerMetrics(out: Exposition, now: Date) {
if (server.outgoingTraffic !== null) out.add("homelab_hetzner_traffic_outgoing_bytes", "gauge", "Outgoing traffic of a Hetzner Cloud server this month, in bytes.", server.outgoingTraffic, labels);
if (server.includedTraffic !== null) out.add("homelab_hetzner_traffic_included_bytes", "gauge", "The outgoing traffic a Hetzner Cloud server includes per month, in bytes.", server.includedTraffic, labels);
}
for (const server of hetznerServersOf(s)) {
const forecast = hetznerTrafficForecast(server, now.getTime());
if (forecast) out.add("homelab_hetzner_traffic_forecast_bytes", "gauge", "What a Hetzner Cloud server's outgoing traffic is forecast to come to by the end of the month, in bytes.", forecast.projectedMonth, { project: row.name, server: server.name });
}
for (const lb of hetznerLoadBalancersOf(s) ?? []) {
out.add("homelab_hetzner_load_balancer_unhealthy_targets", "gauge", "How many targets of a Hetzner Cloud load balancer are unhealthy.", lb.targets.filter((t) => t.health === "unhealthy").length, { project: row.name, load_balancer: lb.name });
}
for (const c of hetznerCertificatesOf(s) ?? []) {
if (c.notValidAfter) out.add("homelab_hetzner_certificate_expiry_timestamp_seconds", "gauge", "When a Hetzner Cloud certificate ends, in seconds since 1970.", Date.parse(c.notValidAfter) / 1000, { project: row.name, certificate: c.name });
}
const cost = hetznerCostOf(s);
if (cost) out.add("homelab_hetzner_monthly_cost", "gauge", "Estimated monthly cost of a Hetzner Cloud project at list prices, including VAT, in the project's currency.", cost.total, { project: row.name, currency: cost.currency });
}
@@ -466,6 +497,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addHetznerMetrics(out, now);
await addSpeedtestMetrics(out, now);
await addFragnetMetrics(out, now);
await addArrMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}
+365
View File
@@ -0,0 +1,365 @@
/**
* Sonarr, Radarr and Prowlarr: reading what their API says (the same one for all three), what is stored, what is wrong (their own health findings, stuck downloads, a disk nearly full), the
* health pass, the alerts, the page's data and /metrics.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-arr-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, arrState } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/arr/adapter.js");
const arr = await import("../src/services/arrSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { arrRouter } = await import("../src/routes/arr.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const HOUR = 60 * MIN;
const GB = 1024 ** 3;
const iso = (ms: number) => new Date(ms).toISOString();
type Item = import("../src/integrations/arr/adapter.js").ArrQueueItem;
type Read = import("../src/integrations/arr/adapter.js").ArrRead;
const item = (extra: Partial<Item> = {}): Item => ({ id: 1, title: "Show S01E01", status: "downloading", tracked: "ok", state: "downloading", messages: [], size: 1000, sizeLeft: 500, added: iso(Date.now() - 30 * MIN), etaAt: null, protocol: "torrent", client: "qBittorrent", indexer: "Tracker", ...extra });
const read = (extra: Partial<Read> = {}): Read => ({ version: "4.0.9", health: [], queue: { total: 0, items: [] }, disks: [{ path: "/tv", label: null, free: 500 * GB, total: 1000 * GB }], missing: 3, indexers: null, ...extra });
// What the applications send.
const apiItem = (id: number, extra: Record<string, unknown> = {}) => ({ id, title: `Show S01E0${id}`, status: "downloading", trackedDownloadStatus: "ok", trackedDownloadState: "downloading", statusMessages: [], size: 1000, sizeleft: 400, added: "2026-06-01T10:00:00Z", estimatedCompletionTime: "2026-06-01T12:00:00Z", protocol: "torrent", downloadClient: "qBittorrent", indexer: "Tracker", ...extra });
describe("what the *arr API says", () => {
test("health: what each check found, with its type; an entry with no message is skipped", () => {
const h = adapter.parseHealth([{ source: "DownloadClientCheck", type: "error", message: "No download client is available", wikiUrl: "x" }, { source: "UpdateCheck", type: "notice", message: "New update is available" }, { source: "Empty", type: "warning" }, "junk"]);
assert.deepEqual(h.map((x) => [x.source, x.type, x.message]), [["DownloadClientCheck", "error", "No download client is available"], ["UpdateCheck", "notice", "New update is available"]]);
assert.deepEqual(adapter.parseHealth({ message: "no" }), []);
});
test("a queue item: its status and how the download is judged, what it says is wrong (the error first), size, times, client and indexer", () => {
const q = adapter.parseQueueItem(apiItem(4, { status: "warning", trackedDownloadStatus: "warning", trackedDownloadState: "importBlocked", errorMessage: "Import failed", statusMessages: [{ title: "Show.S01E04.mkv", messages: ["Not an upgrade", "Not an upgrade"] }, { title: "No files found", messages: [] }] }))!;
assert.deepEqual([q.id, q.status, q.tracked, q.state, q.size, q.sizeLeft, q.client, q.indexer, q.added, q.etaAt], [4, "warning", "warning", "importBlocked", 1000, 400, "qBittorrent", "Tracker", "2026-06-01T10:00:00.000Z", "2026-06-01T12:00:00.000Z"]);
assert.deepEqual(q.messages, ["Import failed", "Show.S01E04.mkv: Not an upgrade", "No files found"]);
});
test("an item without an id is skipped; a page is read with the total the application says", () => {
assert.equal(adapter.parseQueueItem({ title: "x" }), null);
const q = adapter.parseQueue({ page: 1, pageSize: 100, totalRecords: 250, records: [apiItem(1), { title: "no id" }, apiItem(2)] });
assert.deepEqual([q.total, q.items.map((i) => i.id)], [250, [1, 2]]);
assert.deepEqual(adapter.parseQueue(null), { total: 0, items: [] });
});
test("disks: where, how much is free and how big", () => {
const d = adapter.parseDisks([{ path: "/tv", label: "TV", freeSpace: 10, totalSpace: 100 }, { path: "/x" }, "junk"]);
assert.deepEqual(d, [{ path: "/tv", label: "TV", free: 10, total: 100 }]);
});
test("Prowlarr's indexers: the ones it has switched off after failures say until when", () => {
const list = adapter.parseIndexers([{ id: 2, name: "B", enable: true, protocol: "torrent" }, { id: 1, name: "A", enable: false, protocol: "usenet" }, { id: 3, name: "C" }], [{ indexerId: 2, disabledTill: "2026-06-02T00:00:00Z", mostRecentFailure: "2026-06-01T00:00:00Z" }]);
assert.deepEqual(list.map((i) => [i.name, i.enabled, i.disabledTill]), [["A", false, null], ["B", true, "2026-06-02T00:00:00.000Z"], ["C", true, null]]);
});
describe("the adapter against applications that answer", () => {
let fake: Server;
let url = "";
let keyOk = true;
let queueStatus = 200;
let appName = "Sonarr";
const seen: { method: string | undefined; path: string; key: string | string[] | undefined }[] = [];
before(async () => {
fake = createHttpServer((req, res) => {
const u = new URL(req.url ?? "", "http://x");
seen.push({ method: req.method, path: u.pathname + u.search, key: req.headers["x-api-key"] });
const send = (code: number, body: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(body));
};
if (!keyOk) return send(401, { error: "Unauthorized" });
const p = u.pathname.replace(/^\/api\/v[13]/, "");
if (p === "/system/status") return send(200, { appName, version: "4.0.9" });
if (p === "/health") return send(200, [{ source: "IndexerStatusCheck", type: "warning", message: "Indexers unavailable due to failures: A" }]);
if (p === "/queue") return queueStatus === 200 ? send(200, { totalRecords: 1, records: [apiItem(1)] }) : send(queueStatus, {});
if (p === "/diskspace") return send(200, [{ path: "/tv", freeSpace: 100, totalSpace: 1000 }]);
if (p === "/wanted/missing") return send(200, { totalRecords: 7, records: [] });
if (p === "/indexer") return send(200, [{ id: 1, name: "A", enable: true, protocol: "torrent" }]);
if (p === "/indexerstatus") return send(200, [{ indexerId: 1, disabledTill: "2026-06-02T00:00:00Z" }]);
send(404, {});
});
await new Promise<void>((resolve) => fake.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}`;
});
after(() => fake.close());
test("Sonarr: version, health, the queue (asked with its own include), the disks and the missing — with the key in X-Api-Key, and only ever GET", async () => {
seen.length = 0;
const r = await adapter.createArrAdapter("sonarr", { url, apiKey: "k1" }).read();
assert.deepEqual([r.version, r.health.length, r.queue?.total, r.disks?.length, r.missing, r.indexers], ["4.0.9", 1, 1, 1, 7, null]);
assert.ok(seen.every((s) => s.method === "GET" && s.key === "k1"));
assert.ok(seen.some((s) => s.path.startsWith("/api/v3/queue?page=1&pageSize=100&includeUnknownSeriesItems=true")));
});
test("Radarr asks for movies; Prowlarr is under v1, has no queue, and gives its indexers", async () => {
seen.length = 0;
await adapter.createArrAdapter("radarr", { url, apiKey: "k" }).read();
assert.ok(seen.some((s) => s.path.includes("includeUnknownMovieItems=true")));
seen.length = 0;
const r = await adapter.createArrAdapter("prowlarr", { url, apiKey: "k" }).read();
assert.deepEqual([r.queue, r.missing, r.indexers?.[0].name, r.indexers?.[0].disabledTill], [null, null, "A", "2026-06-02T00:00:00.000Z"]);
assert.ok(seen.every((s) => s.path.startsWith("/api/v1/")));
assert.ok(!seen.some((s) => s.path.includes("/queue")));
});
test("a part that cannot be read is null, not empty; the status and the health are what must work", async () => {
queueStatus = 500;
const r = await adapter.createArrAdapter("sonarr", { url, apiKey: "k" }).read();
assert.deepEqual([r.queue, r.disks?.length], [null, 1]);
queueStatus = 200;
});
test("a key that is refused, an address that is not the application, and the wrong kind are said", async () => {
keyOk = false;
const ping = await adapter.createArrAdapter("sonarr", { url, apiKey: "bad" }).ping();
assert.equal(ping.ok, false);
assert.match(ping.error ?? "", /Sonarr did not accept the API key \(HTTP 401\)/);
keyOk = true;
appName = "Radarr";
assert.match((await adapter.createArrAdapter("sonarr", { url, apiKey: "k" }).ping()).error ?? "", /That is Radarr, not Sonarr/);
appName = "Sonarr";
assert.equal((await adapter.createArrAdapter("sonarr", { url, apiKey: "k" }).ping()).ok, true);
assert.equal((await adapter.createArrAdapter("sonarr", { url: "http://127.0.0.1:1", apiKey: "k" }).ping()).ok, false);
await assert.rejects(() => adapter.createArrAdapter("sonarr", { url: `${url}/nowhere`, apiKey: "k" }).read(), /no API at this address/);
});
});
});
describe("what is wrong", () => {
const NOW = Date.parse("2026-06-10T12:00:00Z");
const state = (extra: Partial<import("../src/services/arrSync.js").StoredArr> = {}) => ({ version: "4", health: [], queue: { total: 0, items: [] }, disks: null, missing: 0, indexers: null, updatedAt: iso(NOW), ...extra });
const keys = (list: ReturnType<typeof arr.arrProblems>) => list.map((p) => `${p.key}=${p.severity}`);
const old = (extra: Partial<Item> = {}) => item({ added: iso(NOW - 3 * HOUR), ...extra });
test("an application that is well says nothing", () => {
assert.deepEqual(arr.arrProblems("sonarr", 1, "TV", state({ queue: { total: 1, items: [item({ added: iso(NOW - 10 * MIN) })] }, disks: [{ path: "/tv", label: null, free: 500 * GB, total: 1000 * GB }] }), NOW), []);
});
describe("what it says itself", () => {
test("a warning or an error is raised, a notice is not; the message says who says it", () => {
const p = arr.arrProblems("sonarr", 1, "TV", state({ health: [{ source: "DownloadClientCheck", type: "error", message: "No download client is available" }, { source: "UpdateCheck", type: "notice", message: "New update available" }, { source: "RootFolderCheck", type: "warning", message: "Missing root folder: /tv " }] }), NOW);
assert.equal(p.length, 2);
assert.match(p[0].message, /^TV: Sonarr says no download client is available$/);
assert.match(p[1].message, /Sonarr says missing root folder: \/tv$/);
assert.ok(p.every((x) => x.severity === "warning" && x.key.startsWith("sonarr-health:1:")));
});
test("a finding keeps its key when only its numbers change, and has another when its words do", () => {
const key = (m: string) => arr.arrProblems("prowlarr", 3, "P", state({ health: [{ source: "IndexerStatusCheck", type: "warning", message: m }] }), NOW)[0].key;
assert.equal(key("3 indexers are unavailable"), key("4 indexers are unavailable"));
assert.notEqual(key("3 indexers are unavailable"), key("Download client missing"));
assert.match(key("x"), /^prowlarr-health:3:indexerstatuscheck-/);
});
});
describe("downloads that are stuck", () => {
test("judged wrong for two hours is stuck; younger is left alone; one judged fine is not stuck however old", () => {
assert.equal(arr.isStuck(old({ tracked: "warning" }), NOW), true);
assert.equal(arr.isStuck(item({ tracked: "warning", added: iso(NOW - 30 * MIN) }), NOW), false);
assert.equal(arr.isStuck(old({ tracked: "ok", state: "downloading" }), NOW), false);
});
test("failed, failing and blocked from importing count, whatever the tracked status says", () => {
for (const state of ["failed", "failedPending", "importBlocked"]) assert.equal(arr.isStuck(old({ tracked: "ok", state }), NOW), true, state);
assert.equal(arr.isStuck(old({ tracked: null, state: null, status: "failed" }), NOW), true);
assert.equal(arr.isStuck(old({ tracked: "error", added: null }), NOW), false, "one with no time cannot be called old");
});
test("one alert for the lot: how many, the first three by name, the oldest's age and why", () => {
const items = [old({ id: 1, title: "A", tracked: "error", messages: ["No files found"], added: iso(NOW - 30 * HOUR) }), old({ id: 2, title: "B", tracked: "warning" }), old({ id: 3, title: "C", state: "failed" }), old({ id: 4, title: "D", tracked: "warning" }), item({ id: 5, title: "fine" })];
const p = arr.arrProblems("radarr", 2, "Films", state({ queue: { total: 5, items } }), NOW);
assert.deepEqual(keys(p), ["radarr-queue:2=warning"]);
assert.equal(p[0].message, "Films: 4 downloads are stuck — A; B; C; and 1 more (the oldest for 30 hours) — No files found");
assert.match(arr.arrProblems("radarr", 2, "Films", state({ queue: { total: 1, items: [items[0]] } }), NOW)[0].message, /^Films: 1 download is stuck — A \(the oldest for 30 hours\)/);
});
test("a queue that could not be read says nothing; Prowlarr has none", () => {
assert.deepEqual(arr.arrProblems("sonarr", 1, "TV", state({ queue: null }), NOW), []);
});
});
describe("disks", () => {
const disk = (path: string, free: number, total = 1000) => ({ path, label: null, free: free * GB, total: total * GB });
test("under 10% free is a warning, under 5% critical; the message says how much", () => {
assert.deepEqual(arr.arrProblems("sonarr", 1, "TV", state({ disks: [disk("/tv", 150)] }), NOW), []);
const warn = arr.arrProblems("sonarr", 1, "TV", state({ disks: [disk("/tv", 80)] }), NOW);
assert.deepEqual(keys(warn), ["sonarr-disk:1:tv=warning"]);
assert.equal(warn[0].message, "TV: /tv has 80.0 GB free of 1000 GB (8%)");
assert.deepEqual(keys(arr.arrProblems("sonarr", 1, "TV", state({ disks: [disk("/tv", 30)] }), NOW)), ["sonarr-disk:1:tv=critical"]);
});
test("folders on one disk are one disk (the shortest path names it); a tiny disk is not a media disk; an unread list says nothing", () => {
const d = arr.distinctDisks([disk("/data/tv/anime", 30), disk("/data", 30), disk("/data/tv", 30), disk("/boot", 0.1, 1), disk("/movies", 600)]);
assert.deepEqual(d.map((x) => x.path), ["/data", "/movies"]);
assert.deepEqual(keys(arr.arrProblems("sonarr", 1, "TV", state({ disks: [disk("/data/tv", 30), disk("/data", 30)] }), NOW)), ["sonarr-disk:1:data=critical"]);
assert.deepEqual(arr.arrProblems("sonarr", 1, "TV", state({ disks: null }), NOW), []);
});
});
});
describe("what is stored", () => {
let id = 0;
before(async () => {
id = (await db.insert(integrations).values({ type: "sonarr", name: "TV", baseUrl: "http://sonarr", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
});
const row = async () => (await db.select().from(arrState).where(eq(arrState.integrationId, id)))[0];
after(async () => {
await arr.forgetArr(id);
await db.delete(integrations).where(eq(integrations.id, id));
});
test("the state is replaced at each read; a part that could not be read is stored as null", async () => {
await arr.storeRead(id, read({ queue: null, disks: null }), new Date());
let s = arr.storedOf(await row());
assert.deepEqual([s.queue, s.disks, s.missing, s.health], [null, null, 3, []]);
await arr.storeRead(id, read({ queue: { total: 1, items: [item()] }, health: [{ source: "x", type: "warning", message: "m" }] }), new Date());
s = arr.storedOf(await row());
assert.deepEqual([s.queue?.total, s.health.length], [1, 1]);
});
test("a read that failed is said, and what was stored is left alone", async () => {
const ok = await arr.syncArr(id, new Date(), async () => read({ version: "kept" }));
assert.equal(ok.ok, true);
const failed = await arr.syncArr(id, new Date(), async () => {
throw new Error("connection refused");
});
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
assert.equal(failed.at, ok.at);
assert.equal(arr.storedOf(await row()).version, "kept");
assert.equal(arr.isArrFresh((await arr.arrSyncStatuses())[id], Date.now()), false);
});
test("the kind is what the read is asked for", async () => {
const asked: string[] = [];
await arr.syncArr(id, new Date(), async (kind) => {
asked.push(kind);
return read();
});
assert.deepEqual(asked, ["sonarr"]);
});
});
describe("the health pass, the page and /metrics", () => {
let tvId = 0;
let pId = 0;
let app: Server;
let base = "";
const fresh = () => setInternalFlag("arrSync", JSON.stringify({ [tvId]: { at: new Date().toISOString(), ok: true, error: null }, [pId]: { at: new Date().toISOString(), ok: true, error: null } }));
const stale = () => setInternalFlag("arrSync", JSON.stringify({ [tvId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null }, [pId]: { at: new Date().toISOString(), ok: true, error: null } }));
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
const mk = async (type: "sonarr" | "prowlarr", name: string) => (await db.insert(integrations).values({ type, name, baseUrl: `http://${name}`, config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
tvId = await mk("sonarr", "TV");
pId = await mk("prowlarr", "Indexers");
const now = Date.now();
await arr.storeRead(
tvId,
read({
health: [{ source: "DownloadClientCheck", type: "error", message: "No download client is available" }],
queue: { total: 3, items: [item({ id: 1, tracked: "error", added: iso(now - 5 * HOUR), messages: ["No files found"] }), item({ id: 2 }), item({ id: 3, state: "importBlocked", added: iso(now - 4 * HOUR) })] },
disks: [{ path: "/tv", label: null, free: 20 * GB, total: 1000 * GB }],
}),
new Date(),
);
await arr.storeRead(pId, read({ queue: null, missing: null, disks: null, indexers: [{ id: 1, name: "A", enabled: true, protocol: "torrent", disabledTill: iso(now + HOUR), lastFailure: iso(now) }], health: [{ source: "IndexerStatusCheck", type: "warning", message: "Indexers unavailable due to failures: A" }] }), new Date());
await fresh();
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/arr", arrRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/arr`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass judges each from what is stored, and holds one whose last read is old", async () => {
let { snapshot, held } = await collectSnapshot();
assert.deepEqual(snapshot.arr?.map((a) => [a.kind, a.integrationId]).sort(), [["prowlarr", pId], ["sonarr", tvId]].sort());
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const mine = list.filter((x) => /^(sonarr|prowlarr)-/.test(x.key));
assert.deepEqual(mine.map((x) => x.key.replace(/^([a-z]+-[a-z]+:[0-9]+).*$/, "$1")).sort(), [`prowlarr-health:${pId}`, `sonarr-disk:${tvId}`, `sonarr-health:${tvId}`, `sonarr-queue:${tvId}`].sort());
const disk = mine.find((x) => x.key === `sonarr-disk:${tvId}:tv`)!;
assert.deepEqual([disk.severity, disk.source], ["critical", `sonarr:${tvId}`]);
for (const k of ["health", "queue", "disk"]) assert.equal(subjectOfConditionKey(`sonarr-${k}:${tvId}:x`), `integration:${tvId}`);
await stale();
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.arr?.map((a) => a.integrationId), held.has(`sonarr:${tvId}`)], [[pId], true]);
await fresh();
});
test("the Alerts page says where to look: the disk under Disk, the rest under Services, and the link is the media page", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const q = report.alerts.find((x) => x.id.endsWith(`sonarr-queue:${tvId}`))!;
const d = report.alerts.find((x) => x.id.endsWith(`sonarr-disk:${tvId}:tv`))!;
assert.deepEqual([q.category, q.source, q.link, q.target?.type, d.category], ["services", "Sonarr", "/media", "integration", "disk"]);
assert.match(q.message ?? "", /2 downloads are stuck/);
});
test("the page: each application with its kind, what it says, its queue (the stuck ones marked), its disks and, for Prowlarr, its indexers", async () => {
const body = (await (await fetch(base)).json()) as { apps: { id: number; kind: string; fresh: boolean; problems: { key: string }[]; state: { stuck: number[]; queue: { total: number } | null; disks: { path: string }[]; indexers: { name: string }[] | null; health: unknown[] } }[] };
const tv = body.apps.find((a) => a.id === tvId)!;
assert.deepEqual([tv.kind, tv.fresh, tv.state.queue?.total, tv.state.stuck.sort(), tv.state.disks.map((d) => d.path), tv.state.health.length], ["sonarr", true, 3, [1, 3], ["/tv"], 1]);
assert.equal(tv.problems.length, 3);
const p = body.apps.find((a) => a.id === pId)!;
assert.deepEqual([p.kind, p.state.queue, p.state.indexers?.map((i) => i.name), p.problems.length], ["prowlarr", null, ["A"], 1]);
await stale();
const old = (await (await fetch(base)).json()) as typeof body;
assert.deepEqual(old.apps.find((a) => a.id === tvId)!.problems, [], "an old reading says nothing is wrong");
await fresh();
});
test("/metrics has the health findings, the queue and what is stuck, the missing and each disk's free space — only while the reading is recent", async () => {
let text = await buildMetrics(new Date());
assert.match(text, /homelab_arr_health_issues\{app="sonarr",name="TV"\} 1/);
assert.match(text, /homelab_arr_queue_items\{app="sonarr",name="TV"\} 3/);
assert.match(text, /homelab_arr_queue_stuck_items\{app="sonarr",name="TV"\} 2/);
assert.match(text, /homelab_arr_wanted_missing\{app="sonarr",name="TV"\} 3/);
assert.match(text, /homelab_arr_disk_free_bytes\{app="sonarr",name="TV",path="\/tv"\} 21474836480/);
assert.doesNotMatch(text, /homelab_arr_queue_items\{app="prowlarr"/, "Prowlarr has no queue");
await stale();
text = await buildMetrics(new Date());
assert.doesNotMatch(text, /homelab_arr_[a-z_]+\{app="sonarr"/);
await fresh();
});
test("taking the integration away takes what was read with it", async () => {
await arr.forgetArr(tvId);
assert.equal((await arr.arrSyncStatuses())[tvId], undefined);
await db.delete(integrations).where(eq(integrations.id, tvId));
assert.deepEqual(await db.select().from(arrState).where(eq(arrState.integrationId, tvId)), []);
});
});
+3 -2
View File
@@ -210,7 +210,7 @@ describe("the cost estimate", () => {
primaryIps: [{ id: 1, ip: "1.1.1.1", type: "ipv4", location: "fsn1", assigned: true }, { id: 2, ip: "1.1.1.2", type: "ipv4", location: "fsn1", assigned: false }],
prices: PRICES,
})!;
assert.deepEqual({ ...c }, { currency: "EUR", servers: 15, backups: 2, volumes: 5, snapshots: 0.12, ips: 1, total: 23.12, unassignedIps: 1, unpriced: 0 });
assert.deepEqual({ ...c }, { currency: "EUR", servers: 15, backups: 2, volumes: 5, snapshots: 0.12, ips: 1, loadBalancers: 0, floatingIps: 0, total: 23.12, unassignedIps: 1, unpriced: 0 });
});
test("without the price list there is no estimate; a server whose price is not known is counted as unpriced", () => {
@@ -251,7 +251,8 @@ describe("what is wrong", () => {
});
test("traffic: a warning from 80% of the month's quota, critical at 100%, in the units the page shows", () => {
assert.deepEqual(problems(stored({ outgoingTraffic: 15 * TB })), []);
assert.deepEqual(keys(problems(stored({ outgoingTraffic: 15 * TB }))), ["hetzner-forecast:1:1=warning"], "75% on the tenth is on course to pass it: the forecast, not yet the traffic alert");
assert.deepEqual(problems(stored({ outgoingTraffic: 5 * TB })), []);
const warn = problems(stored({ outgoingTraffic: 17 * TB }));
assert.deepEqual(keys(warn), ["hetzner-traffic:1:1=warning"]);
assert.match(warn[0].message, /web has sent 17\.0 TB of its 20\.0 TB traffic this month \(85%\)/);
+390
View File
@@ -0,0 +1,390 @@
/**
* Hetzner Cloud, the second round: load balancers (and the health of their targets), floating IPs, certificates (and when they end), their share of the cost, and the traffic forecast
* ("at this rate the server passes its quota on the 24th"). What the API says, what is stored, what is wrong, the page's data, the health pass and /metrics.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-hetzner2-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/hetzner/adapter.js");
const hz = await import("../src/services/hetznerSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { hetznerRouter } = await import("../src/routes/hetzner.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
await runMigrations();
const MIN = 60_000;
const HOUR = 60 * MIN;
const DAY = 24 * HOUR;
const TB = 1024 ** 4;
const iso = (ms: number) => new Date(ms).toISOString();
type Server_ = import("../src/integrations/hetzner/adapter.js").HetznerServer;
type Read = import("../src/integrations/hetzner/adapter.js").HetznerRead;
type Lb = import("../src/integrations/hetzner/adapter.js").HetznerLoadBalancer;
type Cert = import("../src/integrations/hetzner/adapter.js").HetznerCertificate;
type Fip = import("../src/integrations/hetzner/adapter.js").HetznerFloatingIp;
const server = (extra: Partial<Server_> = {}): Server_ => ({ id: 1, name: "web", status: "running", created: iso(Date.now() - 30 * DAY), type: "cx22", cores: 2, memoryGb: 4, diskGb: 40, location: "fsn1", city: "Falkenstein", os: "Ubuntu 24.04", ipv4: "203.0.113.5", ipv6: null, privateIps: [], locked: false, rescueEnabled: false, backupWindow: null, includedTraffic: 20 * TB, outgoingTraffic: 1 * TB, ingoingTraffic: 2 * TB, monthlyPrice: 4, ...extra });
const lb = (extra: Partial<Lb> = {}): Lb => ({ id: 10, name: "front", type: "lb11", location: "fsn1", ipv4: "203.0.113.50", ipv6: null, algorithm: "round_robin", services: 1, targets: [{ kind: "server", serverId: 1, label: null, health: "healthy" }], includedTraffic: 20 * TB, outgoingTraffic: 0, monthlyPrice: 6, ...extra });
const cert = (extra: Partial<Cert> = {}): Cert => ({ id: 20, name: "site", type: "managed", domains: ["example.se", "www.example.se"], notValidAfter: iso(Date.now() + 60 * DAY), issuance: "completed", renewal: "completed", error: null, usedBy: 1, ...extra });
const fip = (extra: Partial<Fip> = {}): Fip => ({ id: 30, name: "vip", ip: "203.0.113.60", type: "ipv4", serverId: 1, location: "fsn1", description: null, ...extra });
const PRICES = { currency: "EUR", imagePerGb: 0.012, volumePerGb: 0.05, backupPercent: 20, primaryIps: [], floatingIps: [{ type: "ipv4", location: "fsn1", monthly: 3.5 }] };
describe("what Hetzner says of load balancers, floating IPs and certificates", () => {
test("load balancers: type and place (and its price there), address, services — and each target's health: unhealthy as soon as one port is", () => {
const [l] = adapter.parseLoadBalancers({
load_balancers: [
{
id: 10,
name: "front",
public_net: { ipv4: { ip: "203.0.113.50" }, ipv6: { ip: "2001:db8::1" } },
load_balancer_type: { name: "lb11", prices: [{ location: "nbg1", price_monthly: { gross: "5.0" } }, { location: "fsn1", price_monthly: { gross: "6.4" } }] },
location: { name: "fsn1" },
algorithm: { type: "round_robin" },
services: [{ listen_port: 80 }, { listen_port: 443 }],
included_traffic: 21_990_232_555_520,
outgoing_traffic: 1000,
targets: [
{ type: "server", server: { id: 1 }, health_status: [{ listen_port: 80, status: "healthy" }, { listen_port: 443, status: "healthy" }] },
{ type: "server", server: { id: 2 }, health_status: [{ listen_port: 80, status: "healthy" }, { listen_port: 443, status: "unhealthy" }] },
{ type: "server", server: { id: 3 }, health_status: [] },
{ type: "ip", ip: { ip: "198.51.100.7" }, health_status: [{ listen_port: 80, status: "unhealthy" }] },
],
},
],
});
assert.deepEqual([l.name, l.type, l.location, l.ipv4, l.algorithm, l.services, l.monthlyPrice, l.includedTraffic], ["front", "lb11", "fsn1", "203.0.113.50", "round_robin", 2, 6.4, 21_990_232_555_520]);
assert.deepEqual(l.targets.map((t) => [t.kind, t.serverId ?? t.label, t.health]), [["server", 1, "healthy"], ["server", 2, "unhealthy"], ["server", 3, "unknown"], ["ip", "198.51.100.7", "unhealthy"]]);
});
test("a label selector is the servers it found (each with its own health), or itself when it found none", () => {
const [l] = adapter.parseLoadBalancers({
load_balancers: [
{
id: 1,
name: "x",
targets: [
{ type: "label_selector", label_selector: { selector: "env=prod" }, targets: [{ type: "server", server: { id: 4 }, health_status: [{ status: "healthy" }] }, { type: "server", server: { id: 5 }, health_status: [{ status: "unhealthy" }] }] },
{ type: "label_selector", label_selector: { selector: "env=none" }, targets: [] },
],
},
],
});
assert.deepEqual(l.targets.map((t) => [t.serverId, t.label, t.health]), [[4, "env=prod", "healthy"], [5, "env=prod", "unhealthy"], [null, "env=none", "unknown"]]);
});
test("without an id or a name, or with nothing to read, a load balancer is skipped and the fields are null", () => {
assert.deepEqual(adapter.parseLoadBalancers({ load_balancers: [{ id: 1 }, { name: "x" }, "junk"] }), []);
const [bare] = adapter.parseLoadBalancers({ load_balancers: [{ id: 2, name: "bare" }] });
assert.deepEqual([bare.type, bare.ipv4, bare.monthlyPrice, bare.targets, bare.services], [null, null, null, [], 0]);
assert.deepEqual(adapter.parseLoadBalancers(null), []);
});
test("floating IPs: the address, what it points at (nothing is none) and where it lives", () => {
const list = adapter.parseFloatingIps({ floating_ips: [{ id: 1, name: "vip", ip: "203.0.113.60", type: "ipv4", server: 5, home_location: { name: "fsn1" }, description: "web" }, { id: 2, ip: "203.0.113.61", type: "ipv4", server: null }, { id: 3, ip: "x", type: "weird" }, { id: 4 }] });
assert.deepEqual(list.map((f) => [f.ip, f.serverId, f.location, f.name]), [["203.0.113.60", 5, "fsn1", "vip"], ["203.0.113.61", null, null, null]]);
});
test("certificates: the domains, when it ends, how its issuing and renewal went, and how many things use it", () => {
const list = adapter.parseCertificates({
certificates: [
{ id: 1, name: "site", type: "managed", domain_names: ["example.se", "www.example.se"], not_valid_after: "2026-09-01T10:00:00+00:00", status: { issuance: "completed", renewal: "scheduled" }, used_by: [{ id: 10, type: "load_balancer" }] },
{ id: 2, name: "new", type: "managed", domain_names: ["a.se"], not_valid_after: null, status: { issuance: "failed", error: { code: "x", message: "DNS did not answer" } }, used_by: [] },
{ id: 3 },
],
});
assert.deepEqual(list.map((c) => [c.name, c.type, c.domains.length, c.notValidAfter, c.issuance, c.error, c.usedBy]), [["site", "managed", 2, "2026-09-01T10:00:00.000Z", "completed", null, 1], ["new", "managed", 1, null, "failed", "DNS did not answer", 0]]);
});
test("the price list: a floating IP's price for each kind and place — or the one price an older list has", () => {
const p = adapter.parsePrices({ pricing: { currency: "EUR", floating_ips: [{ type: "ipv4", prices: [{ location: "fsn1", price_monthly: { gross: "3.5700" } }] }] } })!;
assert.deepEqual(p.floatingIps, [{ type: "ipv4", location: "fsn1", monthly: 3.57 }]);
const old = adapter.parsePrices({ pricing: { currency: "EUR", floating_ip: { price_monthly: { gross: "3.1" } } } })!;
assert.deepEqual(old.floatingIps.map((f) => [f.type, f.location, f.monthly]), [["ipv4", "*", 3.1], ["ipv6", "*", 3.1]]);
});
describe("the adapter against a Hetzner that answers", () => {
let fake: Server;
let url = "";
let lbStatus = 200;
before(async () => {
fake = createHttpServer((req, res) => {
const u = new URL(req.url ?? "", "http://x");
const send = (code: number, body: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(body));
};
const page = { meta: { pagination: { page: 1, next_page: null } } };
if (u.pathname === "/v1/servers") return send(200, { servers: [{ id: 1, name: "web" }], ...page });
if (u.pathname === "/v1/load_balancers") return lbStatus === 200 ? send(200, { load_balancers: [{ id: 10, name: "front", targets: [] }], ...page }) : send(lbStatus, {});
if (u.pathname === "/v1/floating_ips") return send(200, { floating_ips: [{ id: 30, ip: "203.0.113.60", type: "ipv4", server: null }], ...page });
if (u.pathname === "/v1/certificates") return send(200, { certificates: [{ id: 20, name: "site", used_by: [] }], ...page });
if (u.pathname === "/v1/pricing") return send(200, { pricing: { currency: "EUR" } });
send(200, { volumes: [], images: [], primary_ips: [], ...page });
});
await new Promise<void>((resolve) => fake.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}/v1`;
});
after(() => fake.close());
test("they are read with the rest; a list that cannot be asked is unknown (null), not empty", async () => {
const r = await adapter.createHetznerAdapter({ token: "t", apiUrl: url }).read();
assert.deepEqual([r.loadBalancers?.length, r.floatingIps?.[0].serverId, r.certificates?.[0].name], [1, null, "site"]);
lbStatus = 500;
const r2 = await adapter.createHetznerAdapter({ token: "t", apiUrl: url }).read();
assert.deepEqual([r2.loadBalancers, r2.floatingIps?.length, r2.servers.length], [null, 1, 1]);
lbStatus = 200;
});
});
});
describe("the cost with load balancers and floating IPs", () => {
test("a load balancer is its type's price, a floating IP its kind's and place's; one without a price is counted as unpriced", () => {
const base = { servers: [server({ monthlyPrice: 10 })], volumes: [], images: [], primaryIps: [], prices: PRICES };
const c = hz.estimateCost({ ...base, loadBalancers: [lb({ monthlyPrice: 6 }), lb({ id: 11, monthlyPrice: null })], floatingIps: [fip(), fip({ id: 31, location: "hel1" })] })!;
assert.deepEqual([c.loadBalancers, c.floatingIps, c.total, c.unpriced], [6, 3.5, 19.5, 2]);
});
test("a list that could not be read adds nothing", () => {
const c = hz.estimateCost({ servers: [server({ monthlyPrice: 10 })], volumes: [], images: [], primaryIps: [], prices: PRICES, loadBalancers: null, floatingIps: null })!;
assert.deepEqual([c.loadBalancers, c.floatingIps, c.total], [0, 0, 10]);
});
test("an older price list's one price counts for every place", () => {
const c = hz.estimateCost({ servers: [], volumes: [], images: [], primaryIps: [], prices: { ...PRICES, floatingIps: [{ type: "ipv4", location: "*", monthly: 3 }] }, floatingIps: [fip({ location: "ash" })] })!;
assert.equal(c.floatingIps, 3);
});
});
describe("the traffic history", () => {
const t0 = new Date("2026-06-10T00:00:00Z");
test("one sample about every six hours, not more often; two weeks are kept", () => {
let h = hz.carryTraffic(1 * TB, [], t0);
h = hz.carryTraffic(1.1 * TB, h, new Date(t0.getTime() + HOUR));
assert.equal(h.length, 1, "a read an hour later adds nothing");
h = hz.carryTraffic(1.2 * TB, h, new Date(t0.getTime() + 7 * HOUR));
assert.equal(h.length, 2);
h = hz.carryTraffic(5 * TB, h, new Date(t0.getTime() + 15 * DAY));
assert.deepEqual(h.map((x) => x.bytes), [5 * TB], "what is over two weeks old is dropped");
});
test("a counter that goes down is a new month: the history starts over", () => {
let h = hz.carryTraffic(18 * TB, [], t0);
h = hz.carryTraffic(0.2 * TB, h, new Date(t0.getTime() + 7 * HOUR));
assert.deepEqual(h.map((x) => x.bytes), [0.2 * TB]);
});
test("a server that said nothing of its traffic keeps what it had", () => {
const h = [{ at: t0.toISOString(), bytes: 1 }];
assert.deepEqual(hz.carryTraffic(null, h, new Date(t0.getTime() + DAY)), h);
});
test("carryTimes keeps it from read to read", () => {
const a = hz.carryTimes([server({ outgoingTraffic: 1 * TB })], [], t0);
const b = hz.carryTimes([server({ outgoingTraffic: 2 * TB })], a, new Date(t0.getTime() + 7 * HOUR));
assert.deepEqual(b[0].trafficHistory?.map((x) => x.bytes), [1 * TB, 2 * TB]);
});
});
describe("the traffic forecast", () => {
const NOW = Date.parse("2026-06-11T00:00:00Z"); // ten days into a month of thirty
const sample = (daysAgo: number, bytes: number) => ({ at: iso(NOW - daysAgo * DAY), bytes });
const s = (used: number, history: { at: string; bytes: number }[] = [], included = 20 * TB) => ({ outgoingTraffic: used, includedTraffic: included, trafficHistory: history });
test("by the month's average when there is no recent history: 10 TB in ten days is 1 TB a day, and 30 TB at the month's end — the quota goes on the 21st", () => {
const f = hz.trafficForecast(s(10 * TB), NOW)!;
assert.equal(f.basis, "month");
assert.equal(Math.round(f.perDay / TB), 1);
assert.equal(Math.round(f.projectedMonth / TB), 30);
assert.equal(f.exhaustedAt && hz.dayText(f.exhaustedAt), "21 June");
});
test("by the last week's rate when there is some: a rate that has gone up counts", () => {
// 4 TB by day 4 (seven days ago is before the month's start, so the sample at day 4 is the oldest in the window), then 10 TB now: 1 TB a day.
const f = hz.trafficForecast(s(10 * TB, [sample(6, 4 * TB), sample(3, 6 * TB)]), NOW)!;
assert.equal(f.basis, "recent");
assert.equal(Math.round(f.perDay / TB), 1);
// The last three days were quicker than the month: 6 -> 10 TB over 3 days is 1.33 TB a day, and that is what the oldest sample in a three-day window gives.
const fast = hz.trafficForecast(s(10 * TB, [sample(3, 6 * TB)]), NOW)!;
assert.equal(fast.basis, "recent");
assert.ok(fast.perDay > 1.2 * TB && fast.perDay < 1.5 * TB);
});
test("a forecast that stays under the quota has no date; a quota already passed has none either", () => {
const calm = hz.trafficForecast(s(2 * TB), NOW)!;
assert.deepEqual([calm.exhaustedAt, Math.round(calm.projectedMonth / TB)], [null, 6]);
assert.equal(hz.trafficForecast(s(21 * TB), NOW)!.exhaustedAt, null);
});
test("too little to say: the first days of a month, a server with no quota or no traffic, a counter that says nothing", () => {
const early = Date.parse("2026-06-02T00:00:00Z");
assert.equal(hz.trafficForecast(s(1 * TB), early), null);
assert.equal(hz.trafficForecast(s(0), NOW), null);
assert.equal(hz.trafficForecast({ outgoingTraffic: null, includedTraffic: 20 * TB }, NOW), null);
assert.equal(hz.trafficForecast({ outgoingTraffic: 5 * TB, includedTraffic: null }, NOW), null);
});
test("samples from last month, or higher than now, are not used", () => {
const f = hz.trafficForecast(s(10 * TB, [sample(20, 19 * TB), sample(5, 30 * TB)]), NOW)!;
assert.equal(f.basis, "month");
});
});
describe("what is wrong, the second round", () => {
const NOW = Date.parse("2026-06-11T00:00:00Z");
const stored = (extra: Partial<Server_> = {}, hist: { at: string; bytes: number }[] = []) => ({ ...server(extra), lastRunningAt: iso(NOW - HOUR), statusSince: iso(NOW - HOUR), rescueSince: null, trafficHistory: hist });
const keys = (list: ReturnType<typeof hz.hetznerProblems>) => list.map((p) => `${p.key}=${p.severity}`);
const problems = (extras: import("../src/services/hetznerSync.js").HetznerExtras, servers = [stored()]) => hz.hetznerProblems(1, "Prod", servers, [], NOW, 48, extras);
describe("load balancers", () => {
test("healthy says nothing; one unhealthy target is a warning naming it; all of them is critical; an unknown is not unhealthy", () => {
assert.deepEqual(problems({ loadBalancers: [lb()] }), []);
const one = problems({ loadBalancers: [lb({ targets: [{ kind: "server", serverId: 1, label: null, health: "healthy" }, { kind: "server", serverId: 2, label: null, health: "unhealthy" }] })] }, [stored(), stored({ id: 2, name: "api" })]);
assert.deepEqual(keys(one), ["hetzner-lb:1:10=warning"]);
assert.match(one[0].message, /Prod: load balancer front has 1 of 2 targets unhealthy \(api\)$/);
const all = problems({ loadBalancers: [lb({ targets: [{ kind: "server", serverId: 1, label: null, health: "unhealthy" }] })] });
assert.deepEqual(keys(all), ["hetzner-lb:1:10=critical"]);
assert.match(all[0].message, /nothing healthy to send to/);
assert.deepEqual(problems({ loadBalancers: [lb({ targets: [{ kind: "server", serverId: 1, label: null, health: "unknown" }] })] }), []);
assert.deepEqual(problems({ loadBalancers: null }), []);
});
});
describe("certificates", () => {
test("far from the end is fine; within two weeks is a warning, within three days critical, past the end critical; the message says who must renew it", () => {
assert.deepEqual(problems({ certificates: [cert()] }), []);
const soon = problems({ certificates: [cert({ notValidAfter: iso(NOW + 10 * DAY) })] });
assert.deepEqual(keys(soon), ["hetzner-cert:1:20=warning"]);
assert.match(soon[0].message, /certificate site \(example\.se, www\.example\.se\) ends in 10 days — Hetzner should have renewed it/);
assert.deepEqual(keys(problems({ certificates: [cert({ notValidAfter: iso(NOW + 2 * DAY), type: "uploaded" })] })), ["hetzner-cert:1:20=critical"]);
assert.match(problems({ certificates: [cert({ notValidAfter: iso(NOW + 2 * DAY), type: "uploaded" })] })[0].message, /uploaded, so it has to be renewed by hand/);
const gone = problems({ certificates: [cert({ notValidAfter: iso(NOW - 3 * DAY) })] });
assert.deepEqual(keys(gone), ["hetzner-cert:1:20=critical"]);
assert.match(gone[0].message, /ended 3 days ago/);
});
test("a managed one that could not be issued or renewed says why; one nothing uses is not news; an unknown end or an unread list says nothing", () => {
const failed = problems({ certificates: [cert({ renewal: "failed", error: "DNS did not answer", notValidAfter: iso(NOW + 59 * DAY + HOUR) })] });
assert.deepEqual(keys(failed), ["hetzner-cert:1:20=warning"]);
assert.match(failed[0].message, /could not be renewed by Hetzner — DNS did not answer \(it is valid for 59 more days\)/);
assert.deepEqual(problems({ certificates: [cert({ notValidAfter: iso(NOW + DAY), usedBy: 0 })] }), []);
assert.deepEqual(problems({ certificates: [cert({ notValidAfter: null })] }), []);
assert.deepEqual(problems({ certificates: null }), []);
});
});
describe("floating IPs", () => {
test("one that points at a server is fine; one that points at none is a warning (it is billed)", () => {
assert.deepEqual(problems({ floatingIps: [fip()] }), []);
const free = problems({ floatingIps: [fip({ serverId: null })] });
assert.deepEqual(keys(free), ["hetzner-fip:1:30=warning"]);
assert.match(free[0].message, /floating IP 203\.0\.113\.60 \(vip\) points at no server — it is billed all the same/);
});
});
describe("the traffic forecast", () => {
test("on course to pass the quota is a warning with the date, the rate and what it is made from; the traffic alert takes over from 80%", () => {
const p = problems({}, [stored({ outgoingTraffic: 10 * TB })]);
assert.deepEqual(keys(p), ["hetzner-forecast:1:1=warning"]);
assert.match(p[0].message, /Prod: at this rate web will pass its 20\.0 TB traffic quota on 21 June — it has sent 10\.0 TB so far, about 1\.0 TB a day \(the month's average\)/);
assert.deepEqual(keys(problems({}, [stored({ outgoingTraffic: 17 * TB })])), ["hetzner-traffic:1:1=warning"], "no forecast beside the traffic alert");
assert.deepEqual(problems({}, [stored({ outgoingTraffic: 3 * TB })]), []);
});
});
});
describe("the health pass, the page and /metrics", () => {
let id = 0;
let app: Server;
let base = "";
const fresh = () => setInternalFlag("hetznerSync", JSON.stringify({ [id]: { at: new Date().toISOString(), ok: true, error: null } }));
const stale = () => setInternalFlag("hetznerSync", JSON.stringify({ [id]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
const sick = lb({ id: 11, name: "api-lb", targets: [{ kind: "server", serverId: 2, label: null, health: "unhealthy" }] });
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
id = (await db.insert(integrations).values({ type: "hetzner", name: "Prod project", baseUrl: "https://api.hetzner.cloud", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
// web has used 11 TB of 20 and the month is a few days old at least: a forecast either way is not what is tested here, so give it nothing.
const read: Read = {
servers: [server({ id: 1, name: "web", outgoingTraffic: 1 * TB }), server({ id: 2, name: "api", ipv4: "203.0.113.9" })],
volumes: [],
images: [],
primaryIps: [],
prices: PRICES,
loadBalancers: [lb(), sick],
floatingIps: [fip(), fip({ id: 31, ip: "203.0.113.61", serverId: null })],
certificates: [cert({ notValidAfter: iso(Date.now() + 5 * DAY) }), cert({ id: 21, name: "unused", notValidAfter: iso(Date.now() + DAY), usedBy: 0 })],
};
await hz.storeRead(id, read, new Date());
await fresh();
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/hetzner", hetznerRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/hetzner`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("they are stored (an unread list as null) and come back", async () => {
const body = (await (await fetch(base)).json()) as { projects: { id: number; state: { loadBalancers: { name: string; targets: { serverName: string | null }[] }[] | null; floatingIps: { ip: string; serverName: string | null }[]; certificates: { name: string }[]; cost: { loadBalancers: number; floatingIps: number }; servers: { name: string; trafficHistory?: unknown; forecast: unknown }[] } }[] };
const p = body.projects.find((x) => x.id === id)!;
assert.deepEqual(p.state.loadBalancers?.map((l) => [l.name, l.targets[0].serverName]), [["front", "web"], ["api-lb", "api"]]);
assert.deepEqual(p.state.floatingIps.map((f) => [f.ip, f.serverName]), [["203.0.113.60", "web"], ["203.0.113.61", null]]);
assert.deepEqual(p.state.certificates.map((c) => c.name), ["site", "unused"]);
assert.deepEqual([p.state.cost.loadBalancers, p.state.cost.floatingIps], [12, 7]);
assert.ok(p.state.servers.every((s) => s.trafficHistory === undefined), "the history is not sent to the page");
});
test("the health pass judges them, and maintenance on the integration silences them", async () => {
const { snapshot } = await collectSnapshot();
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const mine = list.filter((x) => /^hetzner-(lb|cert|fip):/.test(x.key)).map((x) => `${x.key}=${x.severity}`).sort();
assert.deepEqual(mine, [`hetzner-cert:${id}:20=warning`, `hetzner-fip:${id}:31=warning`, `hetzner-lb:${id}:11=critical`].sort(), "the certificate nothing uses is left out");
for (const k of ["lb", "cert", "fip", "forecast"]) assert.equal(subjectOfConditionKey(`hetzner-${k}:${id}:5`), `integration:${id}`);
});
test("the Alerts page says where to look", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const find = (kind: string, n: number) => report.alerts.find((x) => x.id.endsWith(`hetzner-${kind}:${id}:${n}`))!;
assert.deepEqual([find("lb", 11).category, find("cert", 20).category, find("fip", 31).category, find("lb", 11).link], ["services", "services", "services", "/hetzner"]);
});
test("/metrics has each load balancer's unhealthy targets and each certificate's end — only while the reading is recent", async () => {
let text = await buildMetrics(new Date());
assert.match(text, /homelab_hetzner_load_balancer_unhealthy_targets\{project="Prod project",load_balancer="api-lb"\} 1/);
assert.match(text, /homelab_hetzner_load_balancer_unhealthy_targets\{project="Prod project",load_balancer="front"\} 0/);
assert.match(text, /homelab_hetzner_certificate_expiry_timestamp_seconds\{project="Prod project",certificate="site"\} \d/);
await stale();
text = await buildMetrics(new Date());
assert.doesNotMatch(text, /homelab_hetzner/);
await fresh();
});
});