Files
bobbanandClaude Sonnet 5.5 d9083adb5b Sonarr/Radarr/Prowlarr, Hetzner load balancers/floating IPs/certificates, traffic forecast
New integration kinds sonarr, radarr and prowlarr (shared adapter and sync, page under
Apps -> Media managers): own health findings, stuck downloads, nearly full disks.
Hetzner now reads load balancers (target health), floating IPs and certificates, includes
them in the cost estimate, and forecasts the month's traffic against the quota.
Migrations 0072 and 0073; docs, metrics and tests included.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-27 20:04:55 +01:00

440 lines
30 KiB
TypeScript

/**
* Hetzner Cloud: reading what its API says (paged, read-only), what is stored and carried between reads, the cost estimate, what is wrong, how a Hetzner server is matched to one the app
* knows, the health pass, the alerts, the page's routes and /metrics.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-hetzner-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, servers, hetznerState } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/hetzner/adapter.js");
const hz = await import("../src/services/hetznerSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { hetznerRouter } = await import("../src/routes/hetzner.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const HOUR = 60 * MIN;
const TB = 1024 ** 4;
const iso = (ms: number) => new Date(ms).toISOString();
type Read = import("../src/integrations/hetzner/adapter.js").HetznerRead;
type Server_ = import("../src/integrations/hetzner/adapter.js").HetznerServer;
const server = (extra: Partial<Server_> = {}): Server_ => ({ id: 1, name: "web", status: "running", created: iso(Date.now() - 30 * 24 * HOUR), type: "cx22", cores: 2, memoryGb: 4, diskGb: 40, location: "fsn1", city: "Falkenstein", os: "Ubuntu 24.04", ipv4: "203.0.113.5", ipv6: "2001:db8::/64", privateIps: ["10.0.0.2"], locked: false, rescueEnabled: false, backupWindow: null, includedTraffic: 20 * TB, outgoingTraffic: 1 * TB, ingoingTraffic: 2 * TB, monthlyPrice: 4, ...extra });
const PRICES = { currency: "EUR", imagePerGb: 0.012, volumePerGb: 0.05, backupPercent: 20, primaryIps: [{ type: "ipv4", location: "fsn1", monthly: 0.5 }] };
const read = (extra: Partial<Read> = {}): Read => ({ servers: [server()], volumes: [], images: [], primaryIps: [], prices: PRICES, ...extra });
// What Hetzner sends (the OpenAPI file's shapes; prices are strings).
const apiServer = (id: number, name: string, extra: Record<string, unknown> = {}) => ({
id,
name,
status: "running",
created: "2026-01-02T03:04:05+00:00",
public_net: { ipv4: { id: 1, ip: `203.0.113.${id}`, blocked: false, dns_ptr: "x" }, ipv6: { id: 2, ip: "2001:db8::/64", blocked: false, dns_ptr: [] }, floating_ips: [], firewalls: [] },
private_net: [{ network: 7, ip: `10.0.0.${id}`, alias_ips: [], mac_address: "86:00:00:aa:bb:cc" }],
server_type: { id: 22, name: "cx22", cores: 2, memory: 4, disk: 40, prices: [{ location: "nbg1", price_monthly: { net: "3.0000", gross: "3.5700" } }, { location: "fsn1", price_monthly: { net: "3.2900", gross: "3.9151" } }] },
location: { id: 1, name: "fsn1", city: "Falkenstein", country: "DE" },
image: { id: 5, type: "system", name: "ubuntu-24.04", description: "Ubuntu 24.04" },
rescue_enabled: false,
locked: false,
backup_window: null,
outgoing_traffic: 1_000_000_000,
ingoing_traffic: 2_000_000_000,
included_traffic: 21_990_232_555_520,
...extra,
});
describe("what Hetzner's API says", () => {
test("servers: status, type, where, the addresses, the traffic, and the price of its type at its location (including VAT)", () => {
const [s] = adapter.parseServers({ servers: [apiServer(5, "web", { backup_window: "22-02", rescue_enabled: true })] });
assert.deepEqual([s.id, s.name, s.status, s.type, s.location, s.city, s.os], [5, "web", "running", "cx22", "fsn1", "Falkenstein", "Ubuntu 24.04"]);
assert.deepEqual([s.ipv4, s.privateIps, s.rescueEnabled, s.backupWindow, s.includedTraffic, s.outgoingTraffic], ["203.0.113.5", ["10.0.0.5"], true, "22-02", 21_990_232_555_520, 1_000_000_000]);
assert.equal(s.monthlyPrice, 3.9151, "the fsn1 price, not the nbg1 one");
});
test("a server without an id or a name is skipped, and missing parts are null, not guessed", () => {
const list = adapter.parseServers({ servers: [{ name: "x" }, { id: 2 }, { id: 3, name: "bare" }, "junk"] });
assert.deepEqual(list.map((s) => s.name), ["bare"]);
assert.deepEqual([list[0].status, list[0].ipv4, list[0].includedTraffic, list[0].monthlyPrice, list[0].backupWindow, list[0].privateIps], ["unknown", null, null, null, null, []]);
assert.deepEqual(adapter.parseServers(null), []);
});
test("volumes: the size, and the server it is attached to (none is none)", () => {
const v = adapter.parseVolumes({ volumes: [{ id: 1, name: "data", size: 100, server: 5, location: { name: "fsn1" }, status: "available" }, { id: 2, name: "old", size: 50, server: null }, { id: 3 }] });
assert.deepEqual(v.map((x) => [x.name, x.size, x.serverId, x.location]), [["data", 100, 5, "fsn1"], ["old", 50, null, null]]);
});
test("snapshots and backups: a backup belongs to bound_to, a snapshot says created_from; system images are not kept", () => {
const list = adapter.parseImages({
images: [
{ id: 1, type: "backup", description: "auto", created: "2026-05-01T00:00:00+00:00", image_size: 3.5, bound_to: 5 },
{ id: 2, type: "snapshot", description: "before upgrade", created: "2026-04-01T00:00:00+00:00", image_size: 8, created_from: { id: 5, name: "web" } },
{ id: 3, type: "system", description: "Ubuntu" },
],
});
assert.deepEqual(list.map((i) => [i.type, i.serverId, i.sizeGb]), [["backup", 5, 3.5], ["snapshot", 5, 8]]);
});
test("the price list: the per-GB prices, the backup percentage and the primary IP prices; something that is not a price list is not read", () => {
const p = adapter.parsePrices({ pricing: { currency: "EUR", image: { price_per_gb_month: { gross: "0.0119" } }, volume: { price_per_gb_month: { gross: "0.0476" } }, server_backup: { percentage: "20" }, primary_ips: [{ type: "ipv4", prices: [{ location: "fsn1", price_monthly: { gross: "0.5950" } }] }] } })!;
assert.deepEqual([p.currency, p.imagePerGb, p.volumePerGb, p.backupPercent, p.primaryIps], ["EUR", 0.0119, 0.0476, 20, [{ type: "ipv4", location: "fsn1", monthly: 0.595 }]]);
assert.equal(adapter.parsePrices({ message: "no" }), null);
});
test("primary IPs: whether one belongs to a server", () => {
const ips = adapter.parsePrimaryIps({ primary_ips: [{ id: 1, ip: "203.0.113.5", type: "ipv4", location: { name: "fsn1" }, assignee_id: 5 }, { id: 2, ip: "203.0.113.6", type: "ipv4", assignee_id: null }, { id: 3, ip: "x", type: "weird" }] });
assert.deepEqual(ips.map((i) => [i.ip, i.assigned]), [["203.0.113.5", true], ["203.0.113.6", false]]);
});
test("the next page is the one the meta says; the last has none", () => {
assert.equal(adapter.nextPage({ meta: { pagination: { page: 1, next_page: 2 } } }), 2);
assert.equal(adapter.nextPage({ meta: { pagination: { page: 2, next_page: null } } }), null);
assert.equal(adapter.nextPage({}), null);
});
describe("the adapter against a Hetzner that answers", () => {
let fake: Server;
let url = "";
let tokenOk = true;
let volumesStatus = 200;
let rateLimited = false;
const seen: { method: string | undefined; path: string; auth: string | undefined }[] = [];
before(async () => {
fake = createHttpServer((req, res) => {
const u = new URL(req.url ?? "", "http://x");
seen.push({ method: req.method, path: u.pathname + u.search, auth: req.headers.authorization });
const send = (code: number, body: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(body));
};
if (!tokenOk) return send(401, { error: { code: "unauthorized" } });
if (rateLimited) return send(429, { error: { code: "rate_limit_exceeded" } });
const page = Number(u.searchParams.get("page") ?? 1);
if (u.pathname === "/v1/servers") return send(200, { servers: [apiServer(page, `s${page}`)], meta: { pagination: { page, next_page: page < 2 ? page + 1 : null } } });
if (u.pathname === "/v1/volumes") return volumesStatus === 200 ? send(200, { volumes: [{ id: 1, name: "data", size: 10, server: 1, status: "available" }], meta: { pagination: { page: 1, next_page: null } } }) : send(volumesStatus, {});
if (u.pathname === "/v1/images") return send(200, { images: u.searchParams.get("type") === "backup" ? [{ id: 9, type: "backup", created: "2026-05-01T00:00:00+00:00", image_size: 2, bound_to: 1 }] : [{ id: 8, type: "snapshot", created: "2026-04-01T00:00:00+00:00", image_size: 5, created_from: { id: 1 } }], meta: { pagination: { page: 1, next_page: null } } });
if (u.pathname === "/v1/primary_ips") return send(200, { primary_ips: [], meta: { pagination: { page: 1, next_page: null } } });
if (u.pathname === "/v1/pricing") return send(200, { pricing: { currency: "EUR", image: { price_per_gb_month: { gross: "0.0119" } }, volume: { price_per_gb_month: { gross: "0.0476" } }, server_backup: { percentage: "20" }, primary_ips: [] } });
send(404, {});
});
await new Promise<void>((resolve) => fake.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}/v1`;
});
after(() => fake.close());
test("every page of the servers is read, with the token as a Bearer, and only ever with GET", async () => {
seen.length = 0;
const r = await adapter.createHetznerAdapter({ token: "tok", apiUrl: url }).read();
assert.deepEqual(r.servers.map((s) => s.name), ["s1", "s2"]);
assert.deepEqual([r.volumes?.length, r.images?.map((i) => i.type).sort(), r.prices?.currency], [1, ["backup", "snapshot"], "EUR"]);
assert.ok(seen.every((s) => s.method === "GET" && s.auth === "Bearer tok"));
assert.ok(seen.some((s) => s.path.startsWith("/v1/servers?per_page=50&page=2")));
});
test("a list that cannot be asked is unknown (null), not empty; the servers are what must work", async () => {
volumesStatus = 500;
const r = await adapter.createHetznerAdapter({ token: "tok", apiUrl: url }).read();
assert.deepEqual([r.volumes, r.servers.length], [null, 2]);
assert.equal(hz.estimateCost(r)?.volumes, 0);
volumesStatus = 200;
});
test("a token that is refused, and a project over its hourly limit, are said", async () => {
tokenOk = false;
const ping = await adapter.createHetznerAdapter({ token: "bad", apiUrl: url }).ping();
assert.equal(ping.ok, false);
assert.match(ping.error ?? "", /did not accept the token/);
tokenOk = true;
rateLimited = true;
await assert.rejects(() => adapter.createHetznerAdapter({ token: "t", apiUrl: url }).read(), /too many requests/);
rateLimited = false;
assert.equal((await adapter.createHetznerAdapter({ token: "t", apiUrl: url }).ping()).ok, true);
assert.equal((await adapter.createHetznerAdapter({ token: "t", apiUrl: "http://127.0.0.1:1/v1" }).ping()).ok, false);
});
});
});
describe("what is carried between reads", () => {
const t0 = new Date("2026-06-01T10:00:00Z");
test("a server first seen starts its clocks now; one seen running remembers when", () => {
const a = hz.carryTimes([server({ status: "running" }), server({ id: 2, name: "off", status: "off" })], [], t0);
assert.deepEqual([a[0].lastRunningAt, a[0].statusSince, a[1].lastRunningAt, a[1].statusSince], [t0.toISOString(), t0.toISOString(), null, t0.toISOString()]);
});
test("when it goes off the last time it ran is kept, and the status clock starts at the first read that showed it", () => {
const t1 = new Date(t0.getTime() + 5 * MIN);
const t2 = new Date(t0.getTime() + 10 * MIN);
const running = hz.carryTimes([server()], [], t0);
const off1 = hz.carryTimes([server({ status: "off" })], running, t1);
const off2 = hz.carryTimes([server({ status: "off" })], off1, t2);
assert.deepEqual([off1[0].lastRunningAt, off1[0].statusSince], [t0.toISOString(), t1.toISOString()]);
assert.deepEqual([off2[0].lastRunningAt, off2[0].statusSince], [t0.toISOString(), t1.toISOString()], "the clock does not restart while the status is the same");
});
test("rescue mode's clock starts when it is first seen on, and stops when it is off", () => {
const t1 = new Date(t0.getTime() + 5 * MIN);
const on = hz.carryTimes([server({ rescueEnabled: true })], hz.carryTimes([server()], [], t0), t0);
const still = hz.carryTimes([server({ rescueEnabled: true })], on, t1);
assert.equal(still[0].rescueSince, t0.toISOString());
assert.equal(hz.carryTimes([server()], still, t1)[0].rescueSince, null);
});
});
describe("the cost estimate", () => {
test("servers, their backups (a percentage of the server), volumes, snapshots and primary IPs, at list prices", () => {
const c = hz.estimateCost({
servers: [server({ monthlyPrice: 10, backupWindow: "22-02" }), server({ id: 2, name: "b", monthlyPrice: 5 })],
volumes: [{ id: 1, name: "v", size: 100, serverId: 1, location: "fsn1", status: "available" }],
images: [{ id: 1, type: "snapshot", description: null, created: null, sizeGb: 10, serverId: 1 }, { id: 2, type: "backup", description: null, created: null, sizeGb: 30, serverId: 1 }],
primaryIps: [{ id: 1, ip: "1.1.1.1", type: "ipv4", location: "fsn1", assigned: true }, { id: 2, ip: "1.1.1.2", type: "ipv4", location: "fsn1", assigned: false }],
prices: PRICES,
})!;
assert.deepEqual({ ...c }, { currency: "EUR", servers: 15, backups: 2, volumes: 5, snapshots: 0.12, ips: 1, loadBalancers: 0, floatingIps: 0, total: 23.12, unassignedIps: 1, unpriced: 0 });
});
test("without the price list there is no estimate; a server whose price is not known is counted as unpriced", () => {
assert.equal(hz.estimateCost(read({ prices: null })), null);
assert.equal(hz.estimateCost(read({ servers: [server({ monthlyPrice: null })] }))?.unpriced, 1);
});
});
describe("what is wrong", () => {
const NOW = Date.parse("2026-06-10T12:00:00Z");
const stored = (extra: Partial<Server_> = {}, times: Partial<import("../src/services/hetznerSync.js").StoredServer> = {}) => ({ ...server({ created: iso(NOW - 30 * 24 * HOUR), ...extra }), lastRunningAt: iso(NOW - HOUR), statusSince: iso(NOW - HOUR), rescueSince: null, ...times });
const keys = (list: ReturnType<typeof hz.hetznerProblems>) => list.map((p) => `${p.key}=${p.severity}`);
const problems = (s: ReturnType<typeof stored>, images: Read["images"] = [], hours = 48) => hz.hetznerProblems(1, "Prod", [s], images, NOW, hours);
test("a server that is running and well says nothing", () => {
assert.deepEqual(problems(stored()), []);
});
test("a server that was running and is off is a warning; one off for a week, or never seen running, is left alone", () => {
assert.deepEqual(keys(problems(stored({ status: "off" }))), ["hetzner-off:1:1=warning"]);
assert.match(problems(stored({ status: "off" }))[0].message, /Prod: server web is off — it was running 1 hour ago \(Hetzner bills a server that is off\)/);
assert.deepEqual(problems(stored({ status: "off" }, { lastRunningAt: iso(NOW - 8 * 24 * HOUR) })), []);
assert.deepEqual(problems(stored({ status: "off" }, { lastRunningAt: null })), []);
});
test("a status that should pass is a problem after half an hour, and not before", () => {
assert.deepEqual(problems(stored({ status: "migrating" }, { statusSince: iso(NOW - 10 * MIN) })), []);
const p = problems(stored({ status: "migrating" }, { statusSince: iso(NOW - 45 * MIN) }));
assert.deepEqual(keys(p), ["hetzner-stuck:1:1=warning"]);
assert.match(p[0].message, /has been "migrating" for 45 minutes/);
});
test("rescue mode left on for an hour is a warning, and says what a reboot would do", () => {
assert.deepEqual(problems(stored({ rescueEnabled: true }, { rescueSince: iso(NOW - 30 * MIN) })), []);
const p = problems(stored({ rescueEnabled: true }, { rescueSince: iso(NOW - 3 * HOUR) }));
assert.deepEqual(keys(p), ["hetzner-rescue:1:1=warning"]);
assert.match(p[0].message, /next reboot would come up in the rescue system/);
});
test("traffic: a warning from 80% of the month's quota, critical at 100%, in the units the page shows", () => {
assert.deepEqual(keys(problems(stored({ outgoingTraffic: 15 * TB }))), ["hetzner-forecast:1:1=warning"], "75% on the tenth is on course to pass it: the forecast, not yet the traffic alert");
assert.deepEqual(problems(stored({ outgoingTraffic: 5 * TB })), []);
const warn = problems(stored({ outgoingTraffic: 17 * TB }));
assert.deepEqual(keys(warn), ["hetzner-traffic:1:1=warning"]);
assert.match(warn[0].message, /web has sent 17\.0 TB of its 20\.0 TB traffic this month \(85%\)/);
assert.deepEqual(keys(problems(stored({ outgoingTraffic: 21 * TB }))), ["hetzner-traffic:1:1=critical"]);
assert.deepEqual(problems(stored({ outgoingTraffic: null })), []);
assert.deepEqual(problems(stored({ includedTraffic: null, outgoingTraffic: 99 * TB })), []);
});
describe("backups", () => {
const backup = (ageHours: number, serverId = 1) => ({ id: 1, type: "backup" as const, description: null, created: iso(NOW - ageHours * HOUR), sizeGb: 2, serverId });
test("enabled and recent is fine; enabled and old, or none for an old server, is a warning; not enabled is not judged", () => {
const on = stored({ backupWindow: "22-02" });
assert.deepEqual(problems(on, [backup(20)]), []);
assert.match(problems(on, [backup(100)])[0].message, /backups are enabled for web but the newest is 4 days old/);
assert.match(problems(on, [])[0].message, /but none has been made/);
assert.deepEqual(problems(on, [backup(100, 99)]).length, 1, "another server's backup does not count");
assert.deepEqual(problems(stored(), []), []);
});
test("a new server has had no time to make one; the limit is the setting, and 0 turns it off; an unknown list says nothing", () => {
const fresh = stored({ backupWindow: "22-02", created: iso(NOW - 5 * HOUR) });
assert.deepEqual(problems(fresh, []), []);
assert.deepEqual(problems(stored({ backupWindow: "22-02" }), [backup(100)], 0), []);
assert.deepEqual(problems(stored({ backupWindow: "22-02" }), [backup(100)], 200), []);
assert.deepEqual(problems(stored({ backupWindow: "22-02" }), null), []);
});
});
});
describe("which of the app's servers a Hetzner server is", () => {
const app = [{ id: 1, name: "web-prod", ips: ["203.0.113.5", "192.168.1.5"] }, { id: 2, name: "Db", ips: [] }, { id: 3, name: "other", ips: ["10.0.0.2"] }];
test("by an address first, then by name; a server the app does not know matches none", () => {
assert.equal(hz.matchAppServer({ name: "anything", ipv4: "203.0.113.5", privateIps: [] }, app)?.id, 1);
assert.equal(hz.matchAppServer({ name: "x", ipv4: null, privateIps: ["10.0.0.2"] }, app)?.id, 3, "a private address counts");
assert.equal(hz.matchAppServer({ name: "db", ipv4: "198.51.100.9", privateIps: [] }, app)?.id, 2, "names are compared without regard to case");
assert.equal(hz.matchAppServer({ name: "web", ipv4: "198.51.100.9", privateIps: [] }, app), null);
});
test("an address wins over a name that belongs to another server", () => {
assert.equal(hz.matchAppServer({ name: "Db", ipv4: "203.0.113.5", privateIps: [] }, app)?.id, 1);
});
});
describe("what is stored", () => {
let id = 0;
before(async () => {
id = (await db.insert(integrations).values({ type: "hetzner", name: "Project", baseUrl: "https://api.hetzner.cloud", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
});
const row = async () => (await db.select().from(hetznerState).where(eq(hetznerState.integrationId, id)))[0];
after(async () => {
await hz.forgetHetzner(id);
await db.delete(integrations).where(eq(integrations.id, id)); // so its servers are not matched by the tests that follow
});
test("the state is replaced at each read, carrying what was seen between them; an unknown list is stored as null", async () => {
const t0 = new Date();
await hz.storeRead(id, read({ volumes: null, prices: null }), t0);
let r = await row();
assert.deepEqual([hz.serversOf(r).length, hz.volumesOf(r), hz.costOf(r)], [1, null, null]);
await hz.storeRead(id, read({ servers: [server({ status: "off" })], volumes: [{ id: 1, name: "v", size: 10, serverId: 1, location: "fsn1", status: "available" }] }), new Date(t0.getTime() + 5 * MIN));
r = await row();
const [s] = hz.serversOf(r);
assert.deepEqual([s.status, s.lastRunningAt, hz.volumesOf(r)?.length, hz.costOf(r)?.currency], ["off", t0.toISOString(), 1, "EUR"]);
});
test("a read that failed is said, and what was stored is left alone", async () => {
const ok = await hz.syncHetzner(id, new Date(), async () => read({ servers: [server({ name: "kept" })] }));
assert.equal(ok.ok, true);
const failed = await hz.syncHetzner(id, new Date(), async () => {
throw new Error("connection refused");
});
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
assert.equal(failed.at, ok.at);
assert.equal(hz.serversOf(await row())[0].name, "kept");
assert.equal(hz.isHetznerFresh((await hz.hetznerSyncStatuses())[id], Date.now()), false);
});
});
describe("the health pass, the page and /metrics", () => {
let hzId = 0;
let app: Server;
let base = "";
const ids: Record<string, number> = {};
const fresh = () => setInternalFlag("hetznerSync", JSON.stringify({ [hzId]: { at: new Date().toISOString(), ok: true, error: null } }));
const stale = () => setInternalFlag("hetznerSync", JSON.stringify({ [hzId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
hzId = (await db.insert(integrations).values({ type: "hetzner", name: "Prod project", baseUrl: "https://api.hetzner.cloud", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
const mk = async (name: string, ips: string[]) => (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, ipAddresses: JSON.stringify(ips) }).returning({ id: servers.id }))[0].id;
ids.web = await mk("web-agent", ["203.0.113.5"]);
ids.other = await mk("elsewhere", ["192.168.9.9"]);
// web: running but nearly out of traffic; db: was running and is now off; blog: rescue left on for hours.
const t0 = new Date(Date.now() - 3 * HOUR);
await hz.storeRead(hzId, read({ servers: [server({ id: 1, name: "web", outgoingTraffic: 19 * TB, backupWindow: "22-02" }), server({ id: 2, name: "db", ipv4: "203.0.113.6", status: "running" }), server({ id: 3, name: "blog", ipv4: "203.0.113.7", rescueEnabled: true })], volumes: [{ id: 1, name: "data", size: 100, serverId: 1, location: "fsn1", status: "available" }, { id: 2, name: "orphan", size: 20, serverId: null, location: "fsn1", status: "available" }], images: [{ id: 1, type: "snapshot", description: "before upgrade", created: iso(Date.now() - 40 * 24 * HOUR), sizeGb: 8, serverId: 1 }] }), t0);
await hz.storeRead(hzId, read({ servers: [server({ id: 1, name: "web", outgoingTraffic: 19 * TB, backupWindow: "22-02" }), server({ id: 2, name: "db", ipv4: "203.0.113.6", status: "off" }), server({ id: 3, name: "blog", ipv4: "203.0.113.7", rescueEnabled: true })], volumes: [{ id: 1, name: "data", size: 100, serverId: 1, location: "fsn1", status: "available" }, { id: 2, name: "orphan", size: 20, serverId: null, location: "fsn1", status: "available" }], images: [{ id: 1, type: "snapshot", description: "before upgrade", created: iso(Date.now() - 40 * 24 * HOUR), sizeGb: 8, serverId: 1 }] }), new Date());
await fresh();
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/hetzner", hetznerRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/hetzner`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
let { snapshot, held } = await collectSnapshot();
assert.deepEqual(snapshot.hetzner?.map((h) => h.integrationId), [hzId]);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const hetzner = list.filter((x) => x.key.startsWith("hetzner-")).map((x) => `${x.key}=${x.severity}`).sort();
assert.deepEqual(hetzner, [`hetzner-backup:${hzId}:1=warning`, `hetzner-off:${hzId}:2=warning`, `hetzner-rescue:${hzId}:3=warning`, `hetzner-traffic:${hzId}:1=warning`].sort());
const off = list.find((x) => x.key === `hetzner-off:${hzId}:2`)!;
assert.equal(off.source, `hetzner:${hzId}`);
for (const k of ["off", "stuck", "rescue", "traffic", "backup"]) assert.equal(subjectOfConditionKey(`hetzner-${k}:${hzId}:2`), `integration:${hzId}`, "maintenance on the project silences its alerts");
await stale();
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.hetzner ?? [], held.has(`hetzner:${hzId}`)], [[], true]);
await fresh();
});
test("the Alerts page shows them with where to look: the backup under Backups, the traffic under Monitoring, the rest under Services", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const find = (kind: string, server: number) => report.alerts.find((x) => x.id.endsWith(`hetzner-${kind}:${hzId}:${server}`))!;
assert.deepEqual([find("off", 2).category, find("off", 2).source, find("off", 2).link, find("off", 2).target?.type], ["services", "Hetzner", "/hetzner", "integration"]);
assert.equal(find("traffic", 1).category, "monitoring");
assert.equal(find("backup", 1).category, "backup");
assert.equal(find("rescue", 3).category, "services");
});
test("the page: each server with the app's server it is (or none), the volumes with what they hang on, the snapshots, the cost and what is wrong", async () => {
const body = (await (await fetch(base)).json()) as { projects: { id: number; fresh: boolean; problems: unknown[]; state: { servers: { name: string; linked: { id: number; name: string } | null }[]; volumes: { name: string; serverName: string | null }[] | null; images: { description: string; serverName: string | null }[]; cost: { currency: string; total: number } } }[] };
const p = body.projects.find((x) => x.id === hzId)!;
assert.equal(p.fresh, true);
assert.deepEqual(p.state.servers.map((s) => [s.name, s.linked?.name ?? null]), [["web", "web-agent"], ["db", null], ["blog", null]]);
assert.deepEqual(p.state.volumes?.map((v) => [v.name, v.serverName]), [["data", "web"], ["orphan", null]]);
assert.deepEqual(p.state.images.map((i) => [i.description, i.serverName]), [["before upgrade", "web"]]);
assert.equal(p.state.cost.currency, "EUR");
assert.equal(p.problems.length, 4);
await stale();
const old = (await (await fetch(base)).json()) as typeof body;
assert.deepEqual(old.projects.find((x) => x.id === hzId)!.problems, [], "an old reading says nothing is wrong");
await fresh();
});
test("the server's own page gets its Hetzner server, with what is wrong with it, and a server that is not one gets none", async () => {
const mine = (await (await fetch(`${base}/for-server/${ids.web}`)).json()) as { match: { integrationName: string; fresh: boolean; server: { name: string; includedTraffic: number }; problems: { key: string }[]; snapshots: number; newestBackup: string | null } | null };
assert.deepEqual([mine.match?.integrationName, mine.match?.server.name, mine.match?.snapshots, mine.match?.newestBackup], ["Prod project", "web", 1, null]);
assert.deepEqual(mine.match?.problems.map((p) => p.key).sort(), [`hetzner-backup:${hzId}:1`, `hetzner-traffic:${hzId}:1`]);
const none = (await (await fetch(`${base}/for-server/${ids.other}`)).json()) as { match: unknown };
assert.equal(none.match, null);
assert.equal((await fetch(`${base}/for-server/999999`)).status, 404);
assert.equal((await fetch(`${base}/for-server/abc`)).status, 400);
});
test("/metrics has whether each server runs, its traffic against the quota and the project's cost — only while the reading is recent", async () => {
let text = await buildMetrics(new Date());
assert.match(text, /homelab_hetzner_server_running\{project="Prod project",server="web"\} 1/);
assert.match(text, /homelab_hetzner_server_running\{project="Prod project",server="db"\} 0/);
assert.match(text, /homelab_hetzner_traffic_included_bytes\{project="Prod project",server="web"\}/);
assert.match(text, /homelab_hetzner_monthly_cost\{[^}]*currency="EUR"[^}]*\} \d/);
await stale();
text = await buildMetrics(new Date());
assert.doesNotMatch(text, /homelab_hetzner/);
await fresh();
});
test("taking the integration away takes what was read with it", async () => {
await hz.forgetHetzner(hzId);
assert.equal((await hz.hetznerSyncStatuses())[hzId], undefined);
await db.delete(integrations).where(eq(integrations.id, hzId));
assert.deepEqual(await db.select().from(hetznerState).where(eq(hetznerState.integrationId, hzId)), []);
});
});