Sonarr/Radarr/Prowlarr, Hetzner load balancers/floating IPs/certificates, traffic forecast
New integration kinds sonarr, radarr and prowlarr (shared adapter and sync, page under Apps -> Media managers): own health findings, stuck downloads, nearly full disks. Hetzner now reads load balancers (target health), floating IPs and certificates, includes them in the cost estimate, and forecasts the month's traffic against the quota. Migrations 0072 and 0073; docs, metrics and tests included. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9a3d5c466a
commit
d9083adb5b
38 files changed
+18374
-42
No files matched your search
@@ -0,0 +1,365 @@
|
||||
/**
|
||||
* Sonarr, Radarr and Prowlarr: reading what their API says (the same one for all three), what is stored, what is wrong (their own health findings, stuck downloads, a disk nearly full), the
|
||||
* health pass, the alerts, the page's data and /metrics.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-arr-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, integrations, arrState } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const adapter = await import("../src/integrations/arr/adapter.js");
|
||||
const arr = await import("../src/services/arrSync.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { arrRouter } = await import("../src/routes/arr.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
const GB = 1024 ** 3;
|
||||
const iso = (ms: number) => new Date(ms).toISOString();
|
||||
|
||||
type Item = import("../src/integrations/arr/adapter.js").ArrQueueItem;
|
||||
type Read = import("../src/integrations/arr/adapter.js").ArrRead;
|
||||
|
||||
const item = (extra: Partial<Item> = {}): Item => ({ id: 1, title: "Show S01E01", status: "downloading", tracked: "ok", state: "downloading", messages: [], size: 1000, sizeLeft: 500, added: iso(Date.now() - 30 * MIN), etaAt: null, protocol: "torrent", client: "qBittorrent", indexer: "Tracker", ...extra });
|
||||
const read = (extra: Partial<Read> = {}): Read => ({ version: "4.0.9", health: [], queue: { total: 0, items: [] }, disks: [{ path: "/tv", label: null, free: 500 * GB, total: 1000 * GB }], missing: 3, indexers: null, ...extra });
|
||||
|
||||
// What the applications send.
|
||||
const apiItem = (id: number, extra: Record<string, unknown> = {}) => ({ id, title: `Show S01E0${id}`, status: "downloading", trackedDownloadStatus: "ok", trackedDownloadState: "downloading", statusMessages: [], size: 1000, sizeleft: 400, added: "2026-06-01T10:00:00Z", estimatedCompletionTime: "2026-06-01T12:00:00Z", protocol: "torrent", downloadClient: "qBittorrent", indexer: "Tracker", ...extra });
|
||||
|
||||
describe("what the *arr API says", () => {
|
||||
test("health: what each check found, with its type; an entry with no message is skipped", () => {
|
||||
const h = adapter.parseHealth([{ source: "DownloadClientCheck", type: "error", message: "No download client is available", wikiUrl: "x" }, { source: "UpdateCheck", type: "notice", message: "New update is available" }, { source: "Empty", type: "warning" }, "junk"]);
|
||||
assert.deepEqual(h.map((x) => [x.source, x.type, x.message]), [["DownloadClientCheck", "error", "No download client is available"], ["UpdateCheck", "notice", "New update is available"]]);
|
||||
assert.deepEqual(adapter.parseHealth({ message: "no" }), []);
|
||||
});
|
||||
|
||||
test("a queue item: its status and how the download is judged, what it says is wrong (the error first), size, times, client and indexer", () => {
|
||||
const q = adapter.parseQueueItem(apiItem(4, { status: "warning", trackedDownloadStatus: "warning", trackedDownloadState: "importBlocked", errorMessage: "Import failed", statusMessages: [{ title: "Show.S01E04.mkv", messages: ["Not an upgrade", "Not an upgrade"] }, { title: "No files found", messages: [] }] }))!;
|
||||
assert.deepEqual([q.id, q.status, q.tracked, q.state, q.size, q.sizeLeft, q.client, q.indexer, q.added, q.etaAt], [4, "warning", "warning", "importBlocked", 1000, 400, "qBittorrent", "Tracker", "2026-06-01T10:00:00.000Z", "2026-06-01T12:00:00.000Z"]);
|
||||
assert.deepEqual(q.messages, ["Import failed", "Show.S01E04.mkv: Not an upgrade", "No files found"]);
|
||||
});
|
||||
|
||||
test("an item without an id is skipped; a page is read with the total the application says", () => {
|
||||
assert.equal(adapter.parseQueueItem({ title: "x" }), null);
|
||||
const q = adapter.parseQueue({ page: 1, pageSize: 100, totalRecords: 250, records: [apiItem(1), { title: "no id" }, apiItem(2)] });
|
||||
assert.deepEqual([q.total, q.items.map((i) => i.id)], [250, [1, 2]]);
|
||||
assert.deepEqual(adapter.parseQueue(null), { total: 0, items: [] });
|
||||
});
|
||||
|
||||
test("disks: where, how much is free and how big", () => {
|
||||
const d = adapter.parseDisks([{ path: "/tv", label: "TV", freeSpace: 10, totalSpace: 100 }, { path: "/x" }, "junk"]);
|
||||
assert.deepEqual(d, [{ path: "/tv", label: "TV", free: 10, total: 100 }]);
|
||||
});
|
||||
|
||||
test("Prowlarr's indexers: the ones it has switched off after failures say until when", () => {
|
||||
const list = adapter.parseIndexers([{ id: 2, name: "B", enable: true, protocol: "torrent" }, { id: 1, name: "A", enable: false, protocol: "usenet" }, { id: 3, name: "C" }], [{ indexerId: 2, disabledTill: "2026-06-02T00:00:00Z", mostRecentFailure: "2026-06-01T00:00:00Z" }]);
|
||||
assert.deepEqual(list.map((i) => [i.name, i.enabled, i.disabledTill]), [["A", false, null], ["B", true, "2026-06-02T00:00:00.000Z"], ["C", true, null]]);
|
||||
});
|
||||
|
||||
describe("the adapter against applications that answer", () => {
|
||||
let fake: Server;
|
||||
let url = "";
|
||||
let keyOk = true;
|
||||
let queueStatus = 200;
|
||||
let appName = "Sonarr";
|
||||
const seen: { method: string | undefined; path: string; key: string | string[] | undefined }[] = [];
|
||||
before(async () => {
|
||||
fake = createHttpServer((req, res) => {
|
||||
const u = new URL(req.url ?? "", "http://x");
|
||||
seen.push({ method: req.method, path: u.pathname + u.search, key: req.headers["x-api-key"] });
|
||||
const send = (code: number, body: unknown) => {
|
||||
res.writeHead(code, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify(body));
|
||||
};
|
||||
if (!keyOk) return send(401, { error: "Unauthorized" });
|
||||
const p = u.pathname.replace(/^\/api\/v[13]/, "");
|
||||
if (p === "/system/status") return send(200, { appName, version: "4.0.9" });
|
||||
if (p === "/health") return send(200, [{ source: "IndexerStatusCheck", type: "warning", message: "Indexers unavailable due to failures: A" }]);
|
||||
if (p === "/queue") return queueStatus === 200 ? send(200, { totalRecords: 1, records: [apiItem(1)] }) : send(queueStatus, {});
|
||||
if (p === "/diskspace") return send(200, [{ path: "/tv", freeSpace: 100, totalSpace: 1000 }]);
|
||||
if (p === "/wanted/missing") return send(200, { totalRecords: 7, records: [] });
|
||||
if (p === "/indexer") return send(200, [{ id: 1, name: "A", enable: true, protocol: "torrent" }]);
|
||||
if (p === "/indexerstatus") return send(200, [{ indexerId: 1, disabledTill: "2026-06-02T00:00:00Z" }]);
|
||||
send(404, {});
|
||||
});
|
||||
await new Promise<void>((resolve) => fake.listen(0, "127.0.0.1", resolve));
|
||||
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => fake.close());
|
||||
|
||||
test("Sonarr: version, health, the queue (asked with its own include), the disks and the missing — with the key in X-Api-Key, and only ever GET", async () => {
|
||||
seen.length = 0;
|
||||
const r = await adapter.createArrAdapter("sonarr", { url, apiKey: "k1" }).read();
|
||||
assert.deepEqual([r.version, r.health.length, r.queue?.total, r.disks?.length, r.missing, r.indexers], ["4.0.9", 1, 1, 1, 7, null]);
|
||||
assert.ok(seen.every((s) => s.method === "GET" && s.key === "k1"));
|
||||
assert.ok(seen.some((s) => s.path.startsWith("/api/v3/queue?page=1&pageSize=100&includeUnknownSeriesItems=true")));
|
||||
});
|
||||
|
||||
test("Radarr asks for movies; Prowlarr is under v1, has no queue, and gives its indexers", async () => {
|
||||
seen.length = 0;
|
||||
await adapter.createArrAdapter("radarr", { url, apiKey: "k" }).read();
|
||||
assert.ok(seen.some((s) => s.path.includes("includeUnknownMovieItems=true")));
|
||||
seen.length = 0;
|
||||
const r = await adapter.createArrAdapter("prowlarr", { url, apiKey: "k" }).read();
|
||||
assert.deepEqual([r.queue, r.missing, r.indexers?.[0].name, r.indexers?.[0].disabledTill], [null, null, "A", "2026-06-02T00:00:00.000Z"]);
|
||||
assert.ok(seen.every((s) => s.path.startsWith("/api/v1/")));
|
||||
assert.ok(!seen.some((s) => s.path.includes("/queue")));
|
||||
});
|
||||
|
||||
test("a part that cannot be read is null, not empty; the status and the health are what must work", async () => {
|
||||
queueStatus = 500;
|
||||
const r = await adapter.createArrAdapter("sonarr", { url, apiKey: "k" }).read();
|
||||
assert.deepEqual([r.queue, r.disks?.length], [null, 1]);
|
||||
queueStatus = 200;
|
||||
});
|
||||
|
||||
test("a key that is refused, an address that is not the application, and the wrong kind are said", async () => {
|
||||
keyOk = false;
|
||||
const ping = await adapter.createArrAdapter("sonarr", { url, apiKey: "bad" }).ping();
|
||||
assert.equal(ping.ok, false);
|
||||
assert.match(ping.error ?? "", /Sonarr did not accept the API key \(HTTP 401\)/);
|
||||
keyOk = true;
|
||||
appName = "Radarr";
|
||||
assert.match((await adapter.createArrAdapter("sonarr", { url, apiKey: "k" }).ping()).error ?? "", /That is Radarr, not Sonarr/);
|
||||
appName = "Sonarr";
|
||||
assert.equal((await adapter.createArrAdapter("sonarr", { url, apiKey: "k" }).ping()).ok, true);
|
||||
assert.equal((await adapter.createArrAdapter("sonarr", { url: "http://127.0.0.1:1", apiKey: "k" }).ping()).ok, false);
|
||||
await assert.rejects(() => adapter.createArrAdapter("sonarr", { url: `${url}/nowhere`, apiKey: "k" }).read(), /no API at this address/);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is wrong", () => {
|
||||
const NOW = Date.parse("2026-06-10T12:00:00Z");
|
||||
const state = (extra: Partial<import("../src/services/arrSync.js").StoredArr> = {}) => ({ version: "4", health: [], queue: { total: 0, items: [] }, disks: null, missing: 0, indexers: null, updatedAt: iso(NOW), ...extra });
|
||||
const keys = (list: ReturnType<typeof arr.arrProblems>) => list.map((p) => `${p.key}=${p.severity}`);
|
||||
const old = (extra: Partial<Item> = {}) => item({ added: iso(NOW - 3 * HOUR), ...extra });
|
||||
|
||||
test("an application that is well says nothing", () => {
|
||||
assert.deepEqual(arr.arrProblems("sonarr", 1, "TV", state({ queue: { total: 1, items: [item({ added: iso(NOW - 10 * MIN) })] }, disks: [{ path: "/tv", label: null, free: 500 * GB, total: 1000 * GB }] }), NOW), []);
|
||||
});
|
||||
|
||||
describe("what it says itself", () => {
|
||||
test("a warning or an error is raised, a notice is not; the message says who says it", () => {
|
||||
const p = arr.arrProblems("sonarr", 1, "TV", state({ health: [{ source: "DownloadClientCheck", type: "error", message: "No download client is available" }, { source: "UpdateCheck", type: "notice", message: "New update available" }, { source: "RootFolderCheck", type: "warning", message: "Missing root folder: /tv " }] }), NOW);
|
||||
assert.equal(p.length, 2);
|
||||
assert.match(p[0].message, /^TV: Sonarr says no download client is available$/);
|
||||
assert.match(p[1].message, /Sonarr says missing root folder: \/tv$/);
|
||||
assert.ok(p.every((x) => x.severity === "warning" && x.key.startsWith("sonarr-health:1:")));
|
||||
});
|
||||
test("a finding keeps its key when only its numbers change, and has another when its words do", () => {
|
||||
const key = (m: string) => arr.arrProblems("prowlarr", 3, "P", state({ health: [{ source: "IndexerStatusCheck", type: "warning", message: m }] }), NOW)[0].key;
|
||||
assert.equal(key("3 indexers are unavailable"), key("4 indexers are unavailable"));
|
||||
assert.notEqual(key("3 indexers are unavailable"), key("Download client missing"));
|
||||
assert.match(key("x"), /^prowlarr-health:3:indexerstatuscheck-/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("downloads that are stuck", () => {
|
||||
test("judged wrong for two hours is stuck; younger is left alone; one judged fine is not stuck however old", () => {
|
||||
assert.equal(arr.isStuck(old({ tracked: "warning" }), NOW), true);
|
||||
assert.equal(arr.isStuck(item({ tracked: "warning", added: iso(NOW - 30 * MIN) }), NOW), false);
|
||||
assert.equal(arr.isStuck(old({ tracked: "ok", state: "downloading" }), NOW), false);
|
||||
});
|
||||
test("failed, failing and blocked from importing count, whatever the tracked status says", () => {
|
||||
for (const state of ["failed", "failedPending", "importBlocked"]) assert.equal(arr.isStuck(old({ tracked: "ok", state }), NOW), true, state);
|
||||
assert.equal(arr.isStuck(old({ tracked: null, state: null, status: "failed" }), NOW), true);
|
||||
assert.equal(arr.isStuck(old({ tracked: "error", added: null }), NOW), false, "one with no time cannot be called old");
|
||||
});
|
||||
test("one alert for the lot: how many, the first three by name, the oldest's age and why", () => {
|
||||
const items = [old({ id: 1, title: "A", tracked: "error", messages: ["No files found"], added: iso(NOW - 30 * HOUR) }), old({ id: 2, title: "B", tracked: "warning" }), old({ id: 3, title: "C", state: "failed" }), old({ id: 4, title: "D", tracked: "warning" }), item({ id: 5, title: "fine" })];
|
||||
const p = arr.arrProblems("radarr", 2, "Films", state({ queue: { total: 5, items } }), NOW);
|
||||
assert.deepEqual(keys(p), ["radarr-queue:2=warning"]);
|
||||
assert.equal(p[0].message, "Films: 4 downloads are stuck — A; B; C; and 1 more (the oldest for 30 hours) — No files found");
|
||||
assert.match(arr.arrProblems("radarr", 2, "Films", state({ queue: { total: 1, items: [items[0]] } }), NOW)[0].message, /^Films: 1 download is stuck — A \(the oldest for 30 hours\)/);
|
||||
});
|
||||
test("a queue that could not be read says nothing; Prowlarr has none", () => {
|
||||
assert.deepEqual(arr.arrProblems("sonarr", 1, "TV", state({ queue: null }), NOW), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("disks", () => {
|
||||
const disk = (path: string, free: number, total = 1000) => ({ path, label: null, free: free * GB, total: total * GB });
|
||||
test("under 10% free is a warning, under 5% critical; the message says how much", () => {
|
||||
assert.deepEqual(arr.arrProblems("sonarr", 1, "TV", state({ disks: [disk("/tv", 150)] }), NOW), []);
|
||||
const warn = arr.arrProblems("sonarr", 1, "TV", state({ disks: [disk("/tv", 80)] }), NOW);
|
||||
assert.deepEqual(keys(warn), ["sonarr-disk:1:tv=warning"]);
|
||||
assert.equal(warn[0].message, "TV: /tv has 80.0 GB free of 1000 GB (8%)");
|
||||
assert.deepEqual(keys(arr.arrProblems("sonarr", 1, "TV", state({ disks: [disk("/tv", 30)] }), NOW)), ["sonarr-disk:1:tv=critical"]);
|
||||
});
|
||||
test("folders on one disk are one disk (the shortest path names it); a tiny disk is not a media disk; an unread list says nothing", () => {
|
||||
const d = arr.distinctDisks([disk("/data/tv/anime", 30), disk("/data", 30), disk("/data/tv", 30), disk("/boot", 0.1, 1), disk("/movies", 600)]);
|
||||
assert.deepEqual(d.map((x) => x.path), ["/data", "/movies"]);
|
||||
assert.deepEqual(keys(arr.arrProblems("sonarr", 1, "TV", state({ disks: [disk("/data/tv", 30), disk("/data", 30)] }), NOW)), ["sonarr-disk:1:data=critical"]);
|
||||
assert.deepEqual(arr.arrProblems("sonarr", 1, "TV", state({ disks: null }), NOW), []);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is stored", () => {
|
||||
let id = 0;
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "sonarr", name: "TV", baseUrl: "http://sonarr", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
});
|
||||
const row = async () => (await db.select().from(arrState).where(eq(arrState.integrationId, id)))[0];
|
||||
after(async () => {
|
||||
await arr.forgetArr(id);
|
||||
await db.delete(integrations).where(eq(integrations.id, id));
|
||||
});
|
||||
|
||||
test("the state is replaced at each read; a part that could not be read is stored as null", async () => {
|
||||
await arr.storeRead(id, read({ queue: null, disks: null }), new Date());
|
||||
let s = arr.storedOf(await row());
|
||||
assert.deepEqual([s.queue, s.disks, s.missing, s.health], [null, null, 3, []]);
|
||||
await arr.storeRead(id, read({ queue: { total: 1, items: [item()] }, health: [{ source: "x", type: "warning", message: "m" }] }), new Date());
|
||||
s = arr.storedOf(await row());
|
||||
assert.deepEqual([s.queue?.total, s.health.length], [1, 1]);
|
||||
});
|
||||
|
||||
test("a read that failed is said, and what was stored is left alone", async () => {
|
||||
const ok = await arr.syncArr(id, new Date(), async () => read({ version: "kept" }));
|
||||
assert.equal(ok.ok, true);
|
||||
const failed = await arr.syncArr(id, new Date(), async () => {
|
||||
throw new Error("connection refused");
|
||||
});
|
||||
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
|
||||
assert.equal(failed.at, ok.at);
|
||||
assert.equal(arr.storedOf(await row()).version, "kept");
|
||||
assert.equal(arr.isArrFresh((await arr.arrSyncStatuses())[id], Date.now()), false);
|
||||
});
|
||||
|
||||
test("the kind is what the read is asked for", async () => {
|
||||
const asked: string[] = [];
|
||||
await arr.syncArr(id, new Date(), async (kind) => {
|
||||
asked.push(kind);
|
||||
return read();
|
||||
});
|
||||
assert.deepEqual(asked, ["sonarr"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the health pass, the page and /metrics", () => {
|
||||
let tvId = 0;
|
||||
let pId = 0;
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const fresh = () => setInternalFlag("arrSync", JSON.stringify({ [tvId]: { at: new Date().toISOString(), ok: true, error: null }, [pId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
const stale = () => setInternalFlag("arrSync", JSON.stringify({ [tvId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null }, [pId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
const mk = async (type: "sonarr" | "prowlarr", name: string) => (await db.insert(integrations).values({ type, name, baseUrl: `http://${name}`, config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
tvId = await mk("sonarr", "TV");
|
||||
pId = await mk("prowlarr", "Indexers");
|
||||
const now = Date.now();
|
||||
await arr.storeRead(
|
||||
tvId,
|
||||
read({
|
||||
health: [{ source: "DownloadClientCheck", type: "error", message: "No download client is available" }],
|
||||
queue: { total: 3, items: [item({ id: 1, tracked: "error", added: iso(now - 5 * HOUR), messages: ["No files found"] }), item({ id: 2 }), item({ id: 3, state: "importBlocked", added: iso(now - 4 * HOUR) })] },
|
||||
disks: [{ path: "/tv", label: null, free: 20 * GB, total: 1000 * GB }],
|
||||
}),
|
||||
new Date(),
|
||||
);
|
||||
await arr.storeRead(pId, read({ queue: null, missing: null, disks: null, indexers: [{ id: 1, name: "A", enabled: true, protocol: "torrent", disabledTill: iso(now + HOUR), lastFailure: iso(now) }], health: [{ source: "IndexerStatusCheck", type: "warning", message: "Indexers unavailable due to failures: A" }] }), new Date());
|
||||
await fresh();
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/arr", arrRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/arr`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("the health pass judges each from what is stored, and holds one whose last read is old", async () => {
|
||||
let { snapshot, held } = await collectSnapshot();
|
||||
assert.deepEqual(snapshot.arr?.map((a) => [a.kind, a.integrationId]).sort(), [["prowlarr", pId], ["sonarr", tvId]].sort());
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const mine = list.filter((x) => /^(sonarr|prowlarr)-/.test(x.key));
|
||||
assert.deepEqual(mine.map((x) => x.key.replace(/^([a-z]+-[a-z]+:[0-9]+).*$/, "$1")).sort(), [`prowlarr-health:${pId}`, `sonarr-disk:${tvId}`, `sonarr-health:${tvId}`, `sonarr-queue:${tvId}`].sort());
|
||||
const disk = mine.find((x) => x.key === `sonarr-disk:${tvId}:tv`)!;
|
||||
assert.deepEqual([disk.severity, disk.source], ["critical", `sonarr:${tvId}`]);
|
||||
for (const k of ["health", "queue", "disk"]) assert.equal(subjectOfConditionKey(`sonarr-${k}:${tvId}:x`), `integration:${tvId}`);
|
||||
await stale();
|
||||
({ snapshot, held } = await collectSnapshot());
|
||||
assert.deepEqual([snapshot.arr?.map((a) => a.integrationId), held.has(`sonarr:${tvId}`)], [[pId], true]);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("the Alerts page says where to look: the disk under Disk, the rest under Services, and the link is the media page", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const q = report.alerts.find((x) => x.id.endsWith(`sonarr-queue:${tvId}`))!;
|
||||
const d = report.alerts.find((x) => x.id.endsWith(`sonarr-disk:${tvId}:tv`))!;
|
||||
assert.deepEqual([q.category, q.source, q.link, q.target?.type, d.category], ["services", "Sonarr", "/media", "integration", "disk"]);
|
||||
assert.match(q.message ?? "", /2 downloads are stuck/);
|
||||
});
|
||||
|
||||
test("the page: each application with its kind, what it says, its queue (the stuck ones marked), its disks and, for Prowlarr, its indexers", async () => {
|
||||
const body = (await (await fetch(base)).json()) as { apps: { id: number; kind: string; fresh: boolean; problems: { key: string }[]; state: { stuck: number[]; queue: { total: number } | null; disks: { path: string }[]; indexers: { name: string }[] | null; health: unknown[] } }[] };
|
||||
const tv = body.apps.find((a) => a.id === tvId)!;
|
||||
assert.deepEqual([tv.kind, tv.fresh, tv.state.queue?.total, tv.state.stuck.sort(), tv.state.disks.map((d) => d.path), tv.state.health.length], ["sonarr", true, 3, [1, 3], ["/tv"], 1]);
|
||||
assert.equal(tv.problems.length, 3);
|
||||
const p = body.apps.find((a) => a.id === pId)!;
|
||||
assert.deepEqual([p.kind, p.state.queue, p.state.indexers?.map((i) => i.name), p.problems.length], ["prowlarr", null, ["A"], 1]);
|
||||
await stale();
|
||||
const old = (await (await fetch(base)).json()) as typeof body;
|
||||
assert.deepEqual(old.apps.find((a) => a.id === tvId)!.problems, [], "an old reading says nothing is wrong");
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("/metrics has the health findings, the queue and what is stuck, the missing and each disk's free space — only while the reading is recent", async () => {
|
||||
let text = await buildMetrics(new Date());
|
||||
assert.match(text, /homelab_arr_health_issues\{app="sonarr",name="TV"\} 1/);
|
||||
assert.match(text, /homelab_arr_queue_items\{app="sonarr",name="TV"\} 3/);
|
||||
assert.match(text, /homelab_arr_queue_stuck_items\{app="sonarr",name="TV"\} 2/);
|
||||
assert.match(text, /homelab_arr_wanted_missing\{app="sonarr",name="TV"\} 3/);
|
||||
assert.match(text, /homelab_arr_disk_free_bytes\{app="sonarr",name="TV",path="\/tv"\} 21474836480/);
|
||||
assert.doesNotMatch(text, /homelab_arr_queue_items\{app="prowlarr"/, "Prowlarr has no queue");
|
||||
await stale();
|
||||
text = await buildMetrics(new Date());
|
||||
assert.doesNotMatch(text, /homelab_arr_[a-z_]+\{app="sonarr"/);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("taking the integration away takes what was read with it", async () => {
|
||||
await arr.forgetArr(tvId);
|
||||
assert.equal((await arr.arrSyncStatuses())[tvId], undefined);
|
||||
await db.delete(integrations).where(eq(integrations.id, tvId));
|
||||
assert.deepEqual(await db.select().from(arrState).where(eq(arrState.integrationId, tvId)), []);
|
||||
});
|
||||
});
|
||||
@@ -210,7 +210,7 @@ describe("the cost estimate", () => {
|
||||
primaryIps: [{ id: 1, ip: "1.1.1.1", type: "ipv4", location: "fsn1", assigned: true }, { id: 2, ip: "1.1.1.2", type: "ipv4", location: "fsn1", assigned: false }],
|
||||
prices: PRICES,
|
||||
})!;
|
||||
assert.deepEqual({ ...c }, { currency: "EUR", servers: 15, backups: 2, volumes: 5, snapshots: 0.12, ips: 1, total: 23.12, unassignedIps: 1, unpriced: 0 });
|
||||
assert.deepEqual({ ...c }, { currency: "EUR", servers: 15, backups: 2, volumes: 5, snapshots: 0.12, ips: 1, loadBalancers: 0, floatingIps: 0, total: 23.12, unassignedIps: 1, unpriced: 0 });
|
||||
});
|
||||
|
||||
test("without the price list there is no estimate; a server whose price is not known is counted as unpriced", () => {
|
||||
@@ -251,7 +251,8 @@ describe("what is wrong", () => {
|
||||
});
|
||||
|
||||
test("traffic: a warning from 80% of the month's quota, critical at 100%, in the units the page shows", () => {
|
||||
assert.deepEqual(problems(stored({ outgoingTraffic: 15 * TB })), []);
|
||||
assert.deepEqual(keys(problems(stored({ outgoingTraffic: 15 * TB }))), ["hetzner-forecast:1:1=warning"], "75% on the tenth is on course to pass it: the forecast, not yet the traffic alert");
|
||||
assert.deepEqual(problems(stored({ outgoingTraffic: 5 * TB })), []);
|
||||
const warn = problems(stored({ outgoingTraffic: 17 * TB }));
|
||||
assert.deepEqual(keys(warn), ["hetzner-traffic:1:1=warning"]);
|
||||
assert.match(warn[0].message, /web has sent 17\.0 TB of its 20\.0 TB traffic this month \(85%\)/);
|
||||
|
||||
@@ -0,0 +1,390 @@
|
||||
/**
|
||||
* Hetzner Cloud, the second round: load balancers (and the health of their targets), floating IPs, certificates (and when they end), their share of the cost, and the traffic forecast
|
||||
* ("at this rate the server passes its quota on the 24th"). What the API says, what is stored, what is wrong, the page's data, the health pass and /metrics.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-hetzner2-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, integrations } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const adapter = await import("../src/integrations/hetzner/adapter.js");
|
||||
const hz = await import("../src/services/hetznerSync.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { hetznerRouter } = await import("../src/routes/hetzner.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
const DAY = 24 * HOUR;
|
||||
const TB = 1024 ** 4;
|
||||
const iso = (ms: number) => new Date(ms).toISOString();
|
||||
|
||||
type Server_ = import("../src/integrations/hetzner/adapter.js").HetznerServer;
|
||||
type Read = import("../src/integrations/hetzner/adapter.js").HetznerRead;
|
||||
type Lb = import("../src/integrations/hetzner/adapter.js").HetznerLoadBalancer;
|
||||
type Cert = import("../src/integrations/hetzner/adapter.js").HetznerCertificate;
|
||||
type Fip = import("../src/integrations/hetzner/adapter.js").HetznerFloatingIp;
|
||||
|
||||
const server = (extra: Partial<Server_> = {}): Server_ => ({ id: 1, name: "web", status: "running", created: iso(Date.now() - 30 * DAY), type: "cx22", cores: 2, memoryGb: 4, diskGb: 40, location: "fsn1", city: "Falkenstein", os: "Ubuntu 24.04", ipv4: "203.0.113.5", ipv6: null, privateIps: [], locked: false, rescueEnabled: false, backupWindow: null, includedTraffic: 20 * TB, outgoingTraffic: 1 * TB, ingoingTraffic: 2 * TB, monthlyPrice: 4, ...extra });
|
||||
const lb = (extra: Partial<Lb> = {}): Lb => ({ id: 10, name: "front", type: "lb11", location: "fsn1", ipv4: "203.0.113.50", ipv6: null, algorithm: "round_robin", services: 1, targets: [{ kind: "server", serverId: 1, label: null, health: "healthy" }], includedTraffic: 20 * TB, outgoingTraffic: 0, monthlyPrice: 6, ...extra });
|
||||
const cert = (extra: Partial<Cert> = {}): Cert => ({ id: 20, name: "site", type: "managed", domains: ["example.se", "www.example.se"], notValidAfter: iso(Date.now() + 60 * DAY), issuance: "completed", renewal: "completed", error: null, usedBy: 1, ...extra });
|
||||
const fip = (extra: Partial<Fip> = {}): Fip => ({ id: 30, name: "vip", ip: "203.0.113.60", type: "ipv4", serverId: 1, location: "fsn1", description: null, ...extra });
|
||||
const PRICES = { currency: "EUR", imagePerGb: 0.012, volumePerGb: 0.05, backupPercent: 20, primaryIps: [], floatingIps: [{ type: "ipv4", location: "fsn1", monthly: 3.5 }] };
|
||||
|
||||
describe("what Hetzner says of load balancers, floating IPs and certificates", () => {
|
||||
test("load balancers: type and place (and its price there), address, services — and each target's health: unhealthy as soon as one port is", () => {
|
||||
const [l] = adapter.parseLoadBalancers({
|
||||
load_balancers: [
|
||||
{
|
||||
id: 10,
|
||||
name: "front",
|
||||
public_net: { ipv4: { ip: "203.0.113.50" }, ipv6: { ip: "2001:db8::1" } },
|
||||
load_balancer_type: { name: "lb11", prices: [{ location: "nbg1", price_monthly: { gross: "5.0" } }, { location: "fsn1", price_monthly: { gross: "6.4" } }] },
|
||||
location: { name: "fsn1" },
|
||||
algorithm: { type: "round_robin" },
|
||||
services: [{ listen_port: 80 }, { listen_port: 443 }],
|
||||
included_traffic: 21_990_232_555_520,
|
||||
outgoing_traffic: 1000,
|
||||
targets: [
|
||||
{ type: "server", server: { id: 1 }, health_status: [{ listen_port: 80, status: "healthy" }, { listen_port: 443, status: "healthy" }] },
|
||||
{ type: "server", server: { id: 2 }, health_status: [{ listen_port: 80, status: "healthy" }, { listen_port: 443, status: "unhealthy" }] },
|
||||
{ type: "server", server: { id: 3 }, health_status: [] },
|
||||
{ type: "ip", ip: { ip: "198.51.100.7" }, health_status: [{ listen_port: 80, status: "unhealthy" }] },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.deepEqual([l.name, l.type, l.location, l.ipv4, l.algorithm, l.services, l.monthlyPrice, l.includedTraffic], ["front", "lb11", "fsn1", "203.0.113.50", "round_robin", 2, 6.4, 21_990_232_555_520]);
|
||||
assert.deepEqual(l.targets.map((t) => [t.kind, t.serverId ?? t.label, t.health]), [["server", 1, "healthy"], ["server", 2, "unhealthy"], ["server", 3, "unknown"], ["ip", "198.51.100.7", "unhealthy"]]);
|
||||
});
|
||||
|
||||
test("a label selector is the servers it found (each with its own health), or itself when it found none", () => {
|
||||
const [l] = adapter.parseLoadBalancers({
|
||||
load_balancers: [
|
||||
{
|
||||
id: 1,
|
||||
name: "x",
|
||||
targets: [
|
||||
{ type: "label_selector", label_selector: { selector: "env=prod" }, targets: [{ type: "server", server: { id: 4 }, health_status: [{ status: "healthy" }] }, { type: "server", server: { id: 5 }, health_status: [{ status: "unhealthy" }] }] },
|
||||
{ type: "label_selector", label_selector: { selector: "env=none" }, targets: [] },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.deepEqual(l.targets.map((t) => [t.serverId, t.label, t.health]), [[4, "env=prod", "healthy"], [5, "env=prod", "unhealthy"], [null, "env=none", "unknown"]]);
|
||||
});
|
||||
|
||||
test("without an id or a name, or with nothing to read, a load balancer is skipped and the fields are null", () => {
|
||||
assert.deepEqual(adapter.parseLoadBalancers({ load_balancers: [{ id: 1 }, { name: "x" }, "junk"] }), []);
|
||||
const [bare] = adapter.parseLoadBalancers({ load_balancers: [{ id: 2, name: "bare" }] });
|
||||
assert.deepEqual([bare.type, bare.ipv4, bare.monthlyPrice, bare.targets, bare.services], [null, null, null, [], 0]);
|
||||
assert.deepEqual(adapter.parseLoadBalancers(null), []);
|
||||
});
|
||||
|
||||
test("floating IPs: the address, what it points at (nothing is none) and where it lives", () => {
|
||||
const list = adapter.parseFloatingIps({ floating_ips: [{ id: 1, name: "vip", ip: "203.0.113.60", type: "ipv4", server: 5, home_location: { name: "fsn1" }, description: "web" }, { id: 2, ip: "203.0.113.61", type: "ipv4", server: null }, { id: 3, ip: "x", type: "weird" }, { id: 4 }] });
|
||||
assert.deepEqual(list.map((f) => [f.ip, f.serverId, f.location, f.name]), [["203.0.113.60", 5, "fsn1", "vip"], ["203.0.113.61", null, null, null]]);
|
||||
});
|
||||
|
||||
test("certificates: the domains, when it ends, how its issuing and renewal went, and how many things use it", () => {
|
||||
const list = adapter.parseCertificates({
|
||||
certificates: [
|
||||
{ id: 1, name: "site", type: "managed", domain_names: ["example.se", "www.example.se"], not_valid_after: "2026-09-01T10:00:00+00:00", status: { issuance: "completed", renewal: "scheduled" }, used_by: [{ id: 10, type: "load_balancer" }] },
|
||||
{ id: 2, name: "new", type: "managed", domain_names: ["a.se"], not_valid_after: null, status: { issuance: "failed", error: { code: "x", message: "DNS did not answer" } }, used_by: [] },
|
||||
{ id: 3 },
|
||||
],
|
||||
});
|
||||
assert.deepEqual(list.map((c) => [c.name, c.type, c.domains.length, c.notValidAfter, c.issuance, c.error, c.usedBy]), [["site", "managed", 2, "2026-09-01T10:00:00.000Z", "completed", null, 1], ["new", "managed", 1, null, "failed", "DNS did not answer", 0]]);
|
||||
});
|
||||
|
||||
test("the price list: a floating IP's price for each kind and place — or the one price an older list has", () => {
|
||||
const p = adapter.parsePrices({ pricing: { currency: "EUR", floating_ips: [{ type: "ipv4", prices: [{ location: "fsn1", price_monthly: { gross: "3.5700" } }] }] } })!;
|
||||
assert.deepEqual(p.floatingIps, [{ type: "ipv4", location: "fsn1", monthly: 3.57 }]);
|
||||
const old = adapter.parsePrices({ pricing: { currency: "EUR", floating_ip: { price_monthly: { gross: "3.1" } } } })!;
|
||||
assert.deepEqual(old.floatingIps.map((f) => [f.type, f.location, f.monthly]), [["ipv4", "*", 3.1], ["ipv6", "*", 3.1]]);
|
||||
});
|
||||
|
||||
describe("the adapter against a Hetzner that answers", () => {
|
||||
let fake: Server;
|
||||
let url = "";
|
||||
let lbStatus = 200;
|
||||
before(async () => {
|
||||
fake = createHttpServer((req, res) => {
|
||||
const u = new URL(req.url ?? "", "http://x");
|
||||
const send = (code: number, body: unknown) => {
|
||||
res.writeHead(code, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify(body));
|
||||
};
|
||||
const page = { meta: { pagination: { page: 1, next_page: null } } };
|
||||
if (u.pathname === "/v1/servers") return send(200, { servers: [{ id: 1, name: "web" }], ...page });
|
||||
if (u.pathname === "/v1/load_balancers") return lbStatus === 200 ? send(200, { load_balancers: [{ id: 10, name: "front", targets: [] }], ...page }) : send(lbStatus, {});
|
||||
if (u.pathname === "/v1/floating_ips") return send(200, { floating_ips: [{ id: 30, ip: "203.0.113.60", type: "ipv4", server: null }], ...page });
|
||||
if (u.pathname === "/v1/certificates") return send(200, { certificates: [{ id: 20, name: "site", used_by: [] }], ...page });
|
||||
if (u.pathname === "/v1/pricing") return send(200, { pricing: { currency: "EUR" } });
|
||||
send(200, { volumes: [], images: [], primary_ips: [], ...page });
|
||||
});
|
||||
await new Promise<void>((resolve) => fake.listen(0, "127.0.0.1", resolve));
|
||||
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}/v1`;
|
||||
});
|
||||
after(() => fake.close());
|
||||
|
||||
test("they are read with the rest; a list that cannot be asked is unknown (null), not empty", async () => {
|
||||
const r = await adapter.createHetznerAdapter({ token: "t", apiUrl: url }).read();
|
||||
assert.deepEqual([r.loadBalancers?.length, r.floatingIps?.[0].serverId, r.certificates?.[0].name], [1, null, "site"]);
|
||||
lbStatus = 500;
|
||||
const r2 = await adapter.createHetznerAdapter({ token: "t", apiUrl: url }).read();
|
||||
assert.deepEqual([r2.loadBalancers, r2.floatingIps?.length, r2.servers.length], [null, 1, 1]);
|
||||
lbStatus = 200;
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("the cost with load balancers and floating IPs", () => {
|
||||
test("a load balancer is its type's price, a floating IP its kind's and place's; one without a price is counted as unpriced", () => {
|
||||
const base = { servers: [server({ monthlyPrice: 10 })], volumes: [], images: [], primaryIps: [], prices: PRICES };
|
||||
const c = hz.estimateCost({ ...base, loadBalancers: [lb({ monthlyPrice: 6 }), lb({ id: 11, monthlyPrice: null })], floatingIps: [fip(), fip({ id: 31, location: "hel1" })] })!;
|
||||
assert.deepEqual([c.loadBalancers, c.floatingIps, c.total, c.unpriced], [6, 3.5, 19.5, 2]);
|
||||
});
|
||||
test("a list that could not be read adds nothing", () => {
|
||||
const c = hz.estimateCost({ servers: [server({ monthlyPrice: 10 })], volumes: [], images: [], primaryIps: [], prices: PRICES, loadBalancers: null, floatingIps: null })!;
|
||||
assert.deepEqual([c.loadBalancers, c.floatingIps, c.total], [0, 0, 10]);
|
||||
});
|
||||
test("an older price list's one price counts for every place", () => {
|
||||
const c = hz.estimateCost({ servers: [], volumes: [], images: [], primaryIps: [], prices: { ...PRICES, floatingIps: [{ type: "ipv4", location: "*", monthly: 3 }] }, floatingIps: [fip({ location: "ash" })] })!;
|
||||
assert.equal(c.floatingIps, 3);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the traffic history", () => {
|
||||
const t0 = new Date("2026-06-10T00:00:00Z");
|
||||
test("one sample about every six hours, not more often; two weeks are kept", () => {
|
||||
let h = hz.carryTraffic(1 * TB, [], t0);
|
||||
h = hz.carryTraffic(1.1 * TB, h, new Date(t0.getTime() + HOUR));
|
||||
assert.equal(h.length, 1, "a read an hour later adds nothing");
|
||||
h = hz.carryTraffic(1.2 * TB, h, new Date(t0.getTime() + 7 * HOUR));
|
||||
assert.equal(h.length, 2);
|
||||
h = hz.carryTraffic(5 * TB, h, new Date(t0.getTime() + 15 * DAY));
|
||||
assert.deepEqual(h.map((x) => x.bytes), [5 * TB], "what is over two weeks old is dropped");
|
||||
});
|
||||
test("a counter that goes down is a new month: the history starts over", () => {
|
||||
let h = hz.carryTraffic(18 * TB, [], t0);
|
||||
h = hz.carryTraffic(0.2 * TB, h, new Date(t0.getTime() + 7 * HOUR));
|
||||
assert.deepEqual(h.map((x) => x.bytes), [0.2 * TB]);
|
||||
});
|
||||
test("a server that said nothing of its traffic keeps what it had", () => {
|
||||
const h = [{ at: t0.toISOString(), bytes: 1 }];
|
||||
assert.deepEqual(hz.carryTraffic(null, h, new Date(t0.getTime() + DAY)), h);
|
||||
});
|
||||
test("carryTimes keeps it from read to read", () => {
|
||||
const a = hz.carryTimes([server({ outgoingTraffic: 1 * TB })], [], t0);
|
||||
const b = hz.carryTimes([server({ outgoingTraffic: 2 * TB })], a, new Date(t0.getTime() + 7 * HOUR));
|
||||
assert.deepEqual(b[0].trafficHistory?.map((x) => x.bytes), [1 * TB, 2 * TB]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the traffic forecast", () => {
|
||||
const NOW = Date.parse("2026-06-11T00:00:00Z"); // ten days into a month of thirty
|
||||
const sample = (daysAgo: number, bytes: number) => ({ at: iso(NOW - daysAgo * DAY), bytes });
|
||||
const s = (used: number, history: { at: string; bytes: number }[] = [], included = 20 * TB) => ({ outgoingTraffic: used, includedTraffic: included, trafficHistory: history });
|
||||
|
||||
test("by the month's average when there is no recent history: 10 TB in ten days is 1 TB a day, and 30 TB at the month's end — the quota goes on the 21st", () => {
|
||||
const f = hz.trafficForecast(s(10 * TB), NOW)!;
|
||||
assert.equal(f.basis, "month");
|
||||
assert.equal(Math.round(f.perDay / TB), 1);
|
||||
assert.equal(Math.round(f.projectedMonth / TB), 30);
|
||||
assert.equal(f.exhaustedAt && hz.dayText(f.exhaustedAt), "21 June");
|
||||
});
|
||||
|
||||
test("by the last week's rate when there is some: a rate that has gone up counts", () => {
|
||||
// 4 TB by day 4 (seven days ago is before the month's start, so the sample at day 4 is the oldest in the window), then 10 TB now: 1 TB a day.
|
||||
const f = hz.trafficForecast(s(10 * TB, [sample(6, 4 * TB), sample(3, 6 * TB)]), NOW)!;
|
||||
assert.equal(f.basis, "recent");
|
||||
assert.equal(Math.round(f.perDay / TB), 1);
|
||||
// The last three days were quicker than the month: 6 -> 10 TB over 3 days is 1.33 TB a day, and that is what the oldest sample in a three-day window gives.
|
||||
const fast = hz.trafficForecast(s(10 * TB, [sample(3, 6 * TB)]), NOW)!;
|
||||
assert.equal(fast.basis, "recent");
|
||||
assert.ok(fast.perDay > 1.2 * TB && fast.perDay < 1.5 * TB);
|
||||
});
|
||||
|
||||
test("a forecast that stays under the quota has no date; a quota already passed has none either", () => {
|
||||
const calm = hz.trafficForecast(s(2 * TB), NOW)!;
|
||||
assert.deepEqual([calm.exhaustedAt, Math.round(calm.projectedMonth / TB)], [null, 6]);
|
||||
assert.equal(hz.trafficForecast(s(21 * TB), NOW)!.exhaustedAt, null);
|
||||
});
|
||||
|
||||
test("too little to say: the first days of a month, a server with no quota or no traffic, a counter that says nothing", () => {
|
||||
const early = Date.parse("2026-06-02T00:00:00Z");
|
||||
assert.equal(hz.trafficForecast(s(1 * TB), early), null);
|
||||
assert.equal(hz.trafficForecast(s(0), NOW), null);
|
||||
assert.equal(hz.trafficForecast({ outgoingTraffic: null, includedTraffic: 20 * TB }, NOW), null);
|
||||
assert.equal(hz.trafficForecast({ outgoingTraffic: 5 * TB, includedTraffic: null }, NOW), null);
|
||||
});
|
||||
|
||||
test("samples from last month, or higher than now, are not used", () => {
|
||||
const f = hz.trafficForecast(s(10 * TB, [sample(20, 19 * TB), sample(5, 30 * TB)]), NOW)!;
|
||||
assert.equal(f.basis, "month");
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is wrong, the second round", () => {
|
||||
const NOW = Date.parse("2026-06-11T00:00:00Z");
|
||||
const stored = (extra: Partial<Server_> = {}, hist: { at: string; bytes: number }[] = []) => ({ ...server(extra), lastRunningAt: iso(NOW - HOUR), statusSince: iso(NOW - HOUR), rescueSince: null, trafficHistory: hist });
|
||||
const keys = (list: ReturnType<typeof hz.hetznerProblems>) => list.map((p) => `${p.key}=${p.severity}`);
|
||||
const problems = (extras: import("../src/services/hetznerSync.js").HetznerExtras, servers = [stored()]) => hz.hetznerProblems(1, "Prod", servers, [], NOW, 48, extras);
|
||||
|
||||
describe("load balancers", () => {
|
||||
test("healthy says nothing; one unhealthy target is a warning naming it; all of them is critical; an unknown is not unhealthy", () => {
|
||||
assert.deepEqual(problems({ loadBalancers: [lb()] }), []);
|
||||
const one = problems({ loadBalancers: [lb({ targets: [{ kind: "server", serverId: 1, label: null, health: "healthy" }, { kind: "server", serverId: 2, label: null, health: "unhealthy" }] })] }, [stored(), stored({ id: 2, name: "api" })]);
|
||||
assert.deepEqual(keys(one), ["hetzner-lb:1:10=warning"]);
|
||||
assert.match(one[0].message, /Prod: load balancer front has 1 of 2 targets unhealthy \(api\)$/);
|
||||
const all = problems({ loadBalancers: [lb({ targets: [{ kind: "server", serverId: 1, label: null, health: "unhealthy" }] })] });
|
||||
assert.deepEqual(keys(all), ["hetzner-lb:1:10=critical"]);
|
||||
assert.match(all[0].message, /nothing healthy to send to/);
|
||||
assert.deepEqual(problems({ loadBalancers: [lb({ targets: [{ kind: "server", serverId: 1, label: null, health: "unknown" }] })] }), []);
|
||||
assert.deepEqual(problems({ loadBalancers: null }), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("certificates", () => {
|
||||
test("far from the end is fine; within two weeks is a warning, within three days critical, past the end critical; the message says who must renew it", () => {
|
||||
assert.deepEqual(problems({ certificates: [cert()] }), []);
|
||||
const soon = problems({ certificates: [cert({ notValidAfter: iso(NOW + 10 * DAY) })] });
|
||||
assert.deepEqual(keys(soon), ["hetzner-cert:1:20=warning"]);
|
||||
assert.match(soon[0].message, /certificate site \(example\.se, www\.example\.se\) ends in 10 days — Hetzner should have renewed it/);
|
||||
assert.deepEqual(keys(problems({ certificates: [cert({ notValidAfter: iso(NOW + 2 * DAY), type: "uploaded" })] })), ["hetzner-cert:1:20=critical"]);
|
||||
assert.match(problems({ certificates: [cert({ notValidAfter: iso(NOW + 2 * DAY), type: "uploaded" })] })[0].message, /uploaded, so it has to be renewed by hand/);
|
||||
const gone = problems({ certificates: [cert({ notValidAfter: iso(NOW - 3 * DAY) })] });
|
||||
assert.deepEqual(keys(gone), ["hetzner-cert:1:20=critical"]);
|
||||
assert.match(gone[0].message, /ended 3 days ago/);
|
||||
});
|
||||
test("a managed one that could not be issued or renewed says why; one nothing uses is not news; an unknown end or an unread list says nothing", () => {
|
||||
const failed = problems({ certificates: [cert({ renewal: "failed", error: "DNS did not answer", notValidAfter: iso(NOW + 59 * DAY + HOUR) })] });
|
||||
assert.deepEqual(keys(failed), ["hetzner-cert:1:20=warning"]);
|
||||
assert.match(failed[0].message, /could not be renewed by Hetzner — DNS did not answer \(it is valid for 59 more days\)/);
|
||||
assert.deepEqual(problems({ certificates: [cert({ notValidAfter: iso(NOW + DAY), usedBy: 0 })] }), []);
|
||||
assert.deepEqual(problems({ certificates: [cert({ notValidAfter: null })] }), []);
|
||||
assert.deepEqual(problems({ certificates: null }), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("floating IPs", () => {
|
||||
test("one that points at a server is fine; one that points at none is a warning (it is billed)", () => {
|
||||
assert.deepEqual(problems({ floatingIps: [fip()] }), []);
|
||||
const free = problems({ floatingIps: [fip({ serverId: null })] });
|
||||
assert.deepEqual(keys(free), ["hetzner-fip:1:30=warning"]);
|
||||
assert.match(free[0].message, /floating IP 203\.0\.113\.60 \(vip\) points at no server — it is billed all the same/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the traffic forecast", () => {
|
||||
test("on course to pass the quota is a warning with the date, the rate and what it is made from; the traffic alert takes over from 80%", () => {
|
||||
const p = problems({}, [stored({ outgoingTraffic: 10 * TB })]);
|
||||
assert.deepEqual(keys(p), ["hetzner-forecast:1:1=warning"]);
|
||||
assert.match(p[0].message, /Prod: at this rate web will pass its 20\.0 TB traffic quota on 21 June — it has sent 10\.0 TB so far, about 1\.0 TB a day \(the month's average\)/);
|
||||
assert.deepEqual(keys(problems({}, [stored({ outgoingTraffic: 17 * TB })])), ["hetzner-traffic:1:1=warning"], "no forecast beside the traffic alert");
|
||||
assert.deepEqual(problems({}, [stored({ outgoingTraffic: 3 * TB })]), []);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("the health pass, the page and /metrics", () => {
|
||||
let id = 0;
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const fresh = () => setInternalFlag("hetznerSync", JSON.stringify({ [id]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
const stale = () => setInternalFlag("hetznerSync", JSON.stringify({ [id]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
|
||||
const sick = lb({ id: 11, name: "api-lb", targets: [{ kind: "server", serverId: 2, label: null, health: "unhealthy" }] });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
id = (await db.insert(integrations).values({ type: "hetzner", name: "Prod project", baseUrl: "https://api.hetzner.cloud", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
// web has used 11 TB of 20 and the month is a few days old at least: a forecast either way is not what is tested here, so give it nothing.
|
||||
const read: Read = {
|
||||
servers: [server({ id: 1, name: "web", outgoingTraffic: 1 * TB }), server({ id: 2, name: "api", ipv4: "203.0.113.9" })],
|
||||
volumes: [],
|
||||
images: [],
|
||||
primaryIps: [],
|
||||
prices: PRICES,
|
||||
loadBalancers: [lb(), sick],
|
||||
floatingIps: [fip(), fip({ id: 31, ip: "203.0.113.61", serverId: null })],
|
||||
certificates: [cert({ notValidAfter: iso(Date.now() + 5 * DAY) }), cert({ id: 21, name: "unused", notValidAfter: iso(Date.now() + DAY), usedBy: 0 })],
|
||||
};
|
||||
await hz.storeRead(id, read, new Date());
|
||||
await fresh();
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/hetzner", hetznerRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/hetzner`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("they are stored (an unread list as null) and come back", async () => {
|
||||
const body = (await (await fetch(base)).json()) as { projects: { id: number; state: { loadBalancers: { name: string; targets: { serverName: string | null }[] }[] | null; floatingIps: { ip: string; serverName: string | null }[]; certificates: { name: string }[]; cost: { loadBalancers: number; floatingIps: number }; servers: { name: string; trafficHistory?: unknown; forecast: unknown }[] } }[] };
|
||||
const p = body.projects.find((x) => x.id === id)!;
|
||||
assert.deepEqual(p.state.loadBalancers?.map((l) => [l.name, l.targets[0].serverName]), [["front", "web"], ["api-lb", "api"]]);
|
||||
assert.deepEqual(p.state.floatingIps.map((f) => [f.ip, f.serverName]), [["203.0.113.60", "web"], ["203.0.113.61", null]]);
|
||||
assert.deepEqual(p.state.certificates.map((c) => c.name), ["site", "unused"]);
|
||||
assert.deepEqual([p.state.cost.loadBalancers, p.state.cost.floatingIps], [12, 7]);
|
||||
assert.ok(p.state.servers.every((s) => s.trafficHistory === undefined), "the history is not sent to the page");
|
||||
});
|
||||
|
||||
test("the health pass judges them, and maintenance on the integration silences them", async () => {
|
||||
const { snapshot } = await collectSnapshot();
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const mine = list.filter((x) => /^hetzner-(lb|cert|fip):/.test(x.key)).map((x) => `${x.key}=${x.severity}`).sort();
|
||||
assert.deepEqual(mine, [`hetzner-cert:${id}:20=warning`, `hetzner-fip:${id}:31=warning`, `hetzner-lb:${id}:11=critical`].sort(), "the certificate nothing uses is left out");
|
||||
for (const k of ["lb", "cert", "fip", "forecast"]) assert.equal(subjectOfConditionKey(`hetzner-${k}:${id}:5`), `integration:${id}`);
|
||||
});
|
||||
|
||||
test("the Alerts page says where to look", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const find = (kind: string, n: number) => report.alerts.find((x) => x.id.endsWith(`hetzner-${kind}:${id}:${n}`))!;
|
||||
assert.deepEqual([find("lb", 11).category, find("cert", 20).category, find("fip", 31).category, find("lb", 11).link], ["services", "services", "services", "/hetzner"]);
|
||||
});
|
||||
|
||||
test("/metrics has each load balancer's unhealthy targets and each certificate's end — only while the reading is recent", async () => {
|
||||
let text = await buildMetrics(new Date());
|
||||
assert.match(text, /homelab_hetzner_load_balancer_unhealthy_targets\{project="Prod project",load_balancer="api-lb"\} 1/);
|
||||
assert.match(text, /homelab_hetzner_load_balancer_unhealthy_targets\{project="Prod project",load_balancer="front"\} 0/);
|
||||
assert.match(text, /homelab_hetzner_certificate_expiry_timestamp_seconds\{project="Prod project",certificate="site"\} \d/);
|
||||
await stale();
|
||||
text = await buildMetrics(new Date());
|
||||
assert.doesNotMatch(text, /homelab_hetzner/);
|
||||
await fresh();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user