/** * Hetzner Cloud: reading what its API says (paged, read-only), what is stored and carried between reads, the cost estimate, what is wrong, how a Hetzner server is matched to one the app * knows, the health pass, the alerts, the page's routes and /metrics. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { createServer as createHttpServer, type Server } from "node:http"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-hetzner-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, integrations, servers, hetznerState } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const adapter = await import("../src/integrations/hetzner/adapter.js"); const hz = await import("../src/services/hetznerSync.js"); const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js"); const { subjectOfConditionKey } = await import("../src/services/maintenance.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { hetznerRouter } = await import("../src/routes/hetzner.js"); const { buildMetrics } = await import("../src/services/metrics.js"); const { setInternalFlag } = await import("../src/services/settingsStore.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); const MIN = 60_000; const HOUR = 60 * MIN; const TB = 1024 ** 4; const iso = (ms: number) => new Date(ms).toISOString(); type Read = import("../src/integrations/hetzner/adapter.js").HetznerRead; type Server_ = import("../src/integrations/hetzner/adapter.js").HetznerServer; const server = (extra: Partial = {}): Server_ => ({ id: 1, name: "web", status: "running", created: iso(Date.now() - 30 * 24 * HOUR), type: "cx22", cores: 2, memoryGb: 4, diskGb: 40, location: "fsn1", city: "Falkenstein", os: "Ubuntu 24.04", ipv4: "203.0.113.5", ipv6: "2001:db8::/64", privateIps: ["10.0.0.2"], locked: false, rescueEnabled: false, backupWindow: null, includedTraffic: 20 * TB, outgoingTraffic: 1 * TB, ingoingTraffic: 2 * TB, monthlyPrice: 4, ...extra }); const PRICES = { currency: "EUR", imagePerGb: 0.012, volumePerGb: 0.05, backupPercent: 20, primaryIps: [{ type: "ipv4", location: "fsn1", monthly: 0.5 }] }; const read = (extra: Partial = {}): Read => ({ servers: [server()], volumes: [], images: [], primaryIps: [], prices: PRICES, ...extra }); // What Hetzner sends (the OpenAPI file's shapes; prices are strings). const apiServer = (id: number, name: string, extra: Record = {}) => ({ id, name, status: "running", created: "2026-01-02T03:04:05+00:00", public_net: { ipv4: { id: 1, ip: `203.0.113.${id}`, blocked: false, dns_ptr: "x" }, ipv6: { id: 2, ip: "2001:db8::/64", blocked: false, dns_ptr: [] }, floating_ips: [], firewalls: [] }, private_net: [{ network: 7, ip: `10.0.0.${id}`, alias_ips: [], mac_address: "86:00:00:aa:bb:cc" }], server_type: { id: 22, name: "cx22", cores: 2, memory: 4, disk: 40, prices: [{ location: "nbg1", price_monthly: { net: "3.0000", gross: "3.5700" } }, { location: "fsn1", price_monthly: { net: "3.2900", gross: "3.9151" } }] }, location: { id: 1, name: "fsn1", city: "Falkenstein", country: "DE" }, image: { id: 5, type: "system", name: "ubuntu-24.04", description: "Ubuntu 24.04" }, rescue_enabled: false, locked: false, backup_window: null, outgoing_traffic: 1_000_000_000, ingoing_traffic: 2_000_000_000, included_traffic: 21_990_232_555_520, ...extra, }); describe("what Hetzner's API says", () => { test("servers: status, type, where, the addresses, the traffic, and the price of its type at its location (including VAT)", () => { const [s] = adapter.parseServers({ servers: [apiServer(5, "web", { backup_window: "22-02", rescue_enabled: true })] }); assert.deepEqual([s.id, s.name, s.status, s.type, s.location, s.city, s.os], [5, "web", "running", "cx22", "fsn1", "Falkenstein", "Ubuntu 24.04"]); assert.deepEqual([s.ipv4, s.privateIps, s.rescueEnabled, s.backupWindow, s.includedTraffic, s.outgoingTraffic], ["203.0.113.5", ["10.0.0.5"], true, "22-02", 21_990_232_555_520, 1_000_000_000]); assert.equal(s.monthlyPrice, 3.9151, "the fsn1 price, not the nbg1 one"); }); test("a server without an id or a name is skipped, and missing parts are null, not guessed", () => { const list = adapter.parseServers({ servers: [{ name: "x" }, { id: 2 }, { id: 3, name: "bare" }, "junk"] }); assert.deepEqual(list.map((s) => s.name), ["bare"]); assert.deepEqual([list[0].status, list[0].ipv4, list[0].includedTraffic, list[0].monthlyPrice, list[0].backupWindow, list[0].privateIps], ["unknown", null, null, null, null, []]); assert.deepEqual(adapter.parseServers(null), []); }); test("volumes: the size, and the server it is attached to (none is none)", () => { const v = adapter.parseVolumes({ volumes: [{ id: 1, name: "data", size: 100, server: 5, location: { name: "fsn1" }, status: "available" }, { id: 2, name: "old", size: 50, server: null }, { id: 3 }] }); assert.deepEqual(v.map((x) => [x.name, x.size, x.serverId, x.location]), [["data", 100, 5, "fsn1"], ["old", 50, null, null]]); }); test("snapshots and backups: a backup belongs to bound_to, a snapshot says created_from; system images are not kept", () => { const list = adapter.parseImages({ images: [ { id: 1, type: "backup", description: "auto", created: "2026-05-01T00:00:00+00:00", image_size: 3.5, bound_to: 5 }, { id: 2, type: "snapshot", description: "before upgrade", created: "2026-04-01T00:00:00+00:00", image_size: 8, created_from: { id: 5, name: "web" } }, { id: 3, type: "system", description: "Ubuntu" }, ], }); assert.deepEqual(list.map((i) => [i.type, i.serverId, i.sizeGb]), [["backup", 5, 3.5], ["snapshot", 5, 8]]); }); test("the price list: the per-GB prices, the backup percentage and the primary IP prices; something that is not a price list is not read", () => { const p = adapter.parsePrices({ pricing: { currency: "EUR", image: { price_per_gb_month: { gross: "0.0119" } }, volume: { price_per_gb_month: { gross: "0.0476" } }, server_backup: { percentage: "20" }, primary_ips: [{ type: "ipv4", prices: [{ location: "fsn1", price_monthly: { gross: "0.5950" } }] }] } })!; assert.deepEqual([p.currency, p.imagePerGb, p.volumePerGb, p.backupPercent, p.primaryIps], ["EUR", 0.0119, 0.0476, 20, [{ type: "ipv4", location: "fsn1", monthly: 0.595 }]]); assert.equal(adapter.parsePrices({ message: "no" }), null); }); test("primary IPs: whether one belongs to a server", () => { const ips = adapter.parsePrimaryIps({ primary_ips: [{ id: 1, ip: "203.0.113.5", type: "ipv4", location: { name: "fsn1" }, assignee_id: 5 }, { id: 2, ip: "203.0.113.6", type: "ipv4", assignee_id: null }, { id: 3, ip: "x", type: "weird" }] }); assert.deepEqual(ips.map((i) => [i.ip, i.assigned]), [["203.0.113.5", true], ["203.0.113.6", false]]); }); test("the next page is the one the meta says; the last has none", () => { assert.equal(adapter.nextPage({ meta: { pagination: { page: 1, next_page: 2 } } }), 2); assert.equal(adapter.nextPage({ meta: { pagination: { page: 2, next_page: null } } }), null); assert.equal(adapter.nextPage({}), null); }); describe("the adapter against a Hetzner that answers", () => { let fake: Server; let url = ""; let tokenOk = true; let volumesStatus = 200; let rateLimited = false; const seen: { method: string | undefined; path: string; auth: string | undefined }[] = []; before(async () => { fake = createHttpServer((req, res) => { const u = new URL(req.url ?? "", "http://x"); seen.push({ method: req.method, path: u.pathname + u.search, auth: req.headers.authorization }); const send = (code: number, body: unknown) => { res.writeHead(code, { "content-type": "application/json" }); res.end(JSON.stringify(body)); }; if (!tokenOk) return send(401, { error: { code: "unauthorized" } }); if (rateLimited) return send(429, { error: { code: "rate_limit_exceeded" } }); const page = Number(u.searchParams.get("page") ?? 1); if (u.pathname === "/v1/servers") return send(200, { servers: [apiServer(page, `s${page}`)], meta: { pagination: { page, next_page: page < 2 ? page + 1 : null } } }); if (u.pathname === "/v1/volumes") return volumesStatus === 200 ? send(200, { volumes: [{ id: 1, name: "data", size: 10, server: 1, status: "available" }], meta: { pagination: { page: 1, next_page: null } } }) : send(volumesStatus, {}); if (u.pathname === "/v1/images") return send(200, { images: u.searchParams.get("type") === "backup" ? [{ id: 9, type: "backup", created: "2026-05-01T00:00:00+00:00", image_size: 2, bound_to: 1 }] : [{ id: 8, type: "snapshot", created: "2026-04-01T00:00:00+00:00", image_size: 5, created_from: { id: 1 } }], meta: { pagination: { page: 1, next_page: null } } }); if (u.pathname === "/v1/primary_ips") return send(200, { primary_ips: [], meta: { pagination: { page: 1, next_page: null } } }); if (u.pathname === "/v1/pricing") return send(200, { pricing: { currency: "EUR", image: { price_per_gb_month: { gross: "0.0119" } }, volume: { price_per_gb_month: { gross: "0.0476" } }, server_backup: { percentage: "20" }, primary_ips: [] } }); send(404, {}); }); await new Promise((resolve) => fake.listen(0, "127.0.0.1", resolve)); url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}/v1`; }); after(() => fake.close()); test("every page of the servers is read, with the token as a Bearer, and only ever with GET", async () => { seen.length = 0; const r = await adapter.createHetznerAdapter({ token: "tok", apiUrl: url }).read(); assert.deepEqual(r.servers.map((s) => s.name), ["s1", "s2"]); assert.deepEqual([r.volumes?.length, r.images?.map((i) => i.type).sort(), r.prices?.currency], [1, ["backup", "snapshot"], "EUR"]); assert.ok(seen.every((s) => s.method === "GET" && s.auth === "Bearer tok")); assert.ok(seen.some((s) => s.path.startsWith("/v1/servers?per_page=50&page=2"))); }); test("a list that cannot be asked is unknown (null), not empty; the servers are what must work", async () => { volumesStatus = 500; const r = await adapter.createHetznerAdapter({ token: "tok", apiUrl: url }).read(); assert.deepEqual([r.volumes, r.servers.length], [null, 2]); assert.equal(hz.estimateCost(r)?.volumes, 0); volumesStatus = 200; }); test("a token that is refused, and a project over its hourly limit, are said", async () => { tokenOk = false; const ping = await adapter.createHetznerAdapter({ token: "bad", apiUrl: url }).ping(); assert.equal(ping.ok, false); assert.match(ping.error ?? "", /did not accept the token/); tokenOk = true; rateLimited = true; await assert.rejects(() => adapter.createHetznerAdapter({ token: "t", apiUrl: url }).read(), /too many requests/); rateLimited = false; assert.equal((await adapter.createHetznerAdapter({ token: "t", apiUrl: url }).ping()).ok, true); assert.equal((await adapter.createHetznerAdapter({ token: "t", apiUrl: "http://127.0.0.1:1/v1" }).ping()).ok, false); }); }); }); describe("what is carried between reads", () => { const t0 = new Date("2026-06-01T10:00:00Z"); test("a server first seen starts its clocks now; one seen running remembers when", () => { const a = hz.carryTimes([server({ status: "running" }), server({ id: 2, name: "off", status: "off" })], [], t0); assert.deepEqual([a[0].lastRunningAt, a[0].statusSince, a[1].lastRunningAt, a[1].statusSince], [t0.toISOString(), t0.toISOString(), null, t0.toISOString()]); }); test("when it goes off the last time it ran is kept, and the status clock starts at the first read that showed it", () => { const t1 = new Date(t0.getTime() + 5 * MIN); const t2 = new Date(t0.getTime() + 10 * MIN); const running = hz.carryTimes([server()], [], t0); const off1 = hz.carryTimes([server({ status: "off" })], running, t1); const off2 = hz.carryTimes([server({ status: "off" })], off1, t2); assert.deepEqual([off1[0].lastRunningAt, off1[0].statusSince], [t0.toISOString(), t1.toISOString()]); assert.deepEqual([off2[0].lastRunningAt, off2[0].statusSince], [t0.toISOString(), t1.toISOString()], "the clock does not restart while the status is the same"); }); test("rescue mode's clock starts when it is first seen on, and stops when it is off", () => { const t1 = new Date(t0.getTime() + 5 * MIN); const on = hz.carryTimes([server({ rescueEnabled: true })], hz.carryTimes([server()], [], t0), t0); const still = hz.carryTimes([server({ rescueEnabled: true })], on, t1); assert.equal(still[0].rescueSince, t0.toISOString()); assert.equal(hz.carryTimes([server()], still, t1)[0].rescueSince, null); }); }); describe("the cost estimate", () => { test("servers, their backups (a percentage of the server), volumes, snapshots and primary IPs, at list prices", () => { const c = hz.estimateCost({ servers: [server({ monthlyPrice: 10, backupWindow: "22-02" }), server({ id: 2, name: "b", monthlyPrice: 5 })], volumes: [{ id: 1, name: "v", size: 100, serverId: 1, location: "fsn1", status: "available" }], images: [{ id: 1, type: "snapshot", description: null, created: null, sizeGb: 10, serverId: 1 }, { id: 2, type: "backup", description: null, created: null, sizeGb: 30, serverId: 1 }], primaryIps: [{ id: 1, ip: "1.1.1.1", type: "ipv4", location: "fsn1", assigned: true }, { id: 2, ip: "1.1.1.2", type: "ipv4", location: "fsn1", assigned: false }], prices: PRICES, })!; assert.deepEqual({ ...c }, { currency: "EUR", servers: 15, backups: 2, volumes: 5, snapshots: 0.12, ips: 1, loadBalancers: 0, floatingIps: 0, total: 23.12, unassignedIps: 1, unpriced: 0 }); }); test("without the price list there is no estimate; a server whose price is not known is counted as unpriced", () => { assert.equal(hz.estimateCost(read({ prices: null })), null); assert.equal(hz.estimateCost(read({ servers: [server({ monthlyPrice: null })] }))?.unpriced, 1); }); }); describe("what is wrong", () => { const NOW = Date.parse("2026-06-10T12:00:00Z"); const stored = (extra: Partial = {}, times: Partial = {}) => ({ ...server({ created: iso(NOW - 30 * 24 * HOUR), ...extra }), lastRunningAt: iso(NOW - HOUR), statusSince: iso(NOW - HOUR), rescueSince: null, ...times }); const keys = (list: ReturnType) => list.map((p) => `${p.key}=${p.severity}`); const problems = (s: ReturnType, images: Read["images"] = [], hours = 48) => hz.hetznerProblems(1, "Prod", [s], images, NOW, hours); test("a server that is running and well says nothing", () => { assert.deepEqual(problems(stored()), []); }); test("a server that was running and is off is a warning; one off for a week, or never seen running, is left alone", () => { assert.deepEqual(keys(problems(stored({ status: "off" }))), ["hetzner-off:1:1=warning"]); assert.match(problems(stored({ status: "off" }))[0].message, /Prod: server web is off — it was running 1 hour ago \(Hetzner bills a server that is off\)/); assert.deepEqual(problems(stored({ status: "off" }, { lastRunningAt: iso(NOW - 8 * 24 * HOUR) })), []); assert.deepEqual(problems(stored({ status: "off" }, { lastRunningAt: null })), []); }); test("a status that should pass is a problem after half an hour, and not before", () => { assert.deepEqual(problems(stored({ status: "migrating" }, { statusSince: iso(NOW - 10 * MIN) })), []); const p = problems(stored({ status: "migrating" }, { statusSince: iso(NOW - 45 * MIN) })); assert.deepEqual(keys(p), ["hetzner-stuck:1:1=warning"]); assert.match(p[0].message, /has been "migrating" for 45 minutes/); }); test("rescue mode left on for an hour is a warning, and says what a reboot would do", () => { assert.deepEqual(problems(stored({ rescueEnabled: true }, { rescueSince: iso(NOW - 30 * MIN) })), []); const p = problems(stored({ rescueEnabled: true }, { rescueSince: iso(NOW - 3 * HOUR) })); assert.deepEqual(keys(p), ["hetzner-rescue:1:1=warning"]); assert.match(p[0].message, /next reboot would come up in the rescue system/); }); test("traffic: a warning from 80% of the month's quota, critical at 100%, in the units the page shows", () => { assert.deepEqual(keys(problems(stored({ outgoingTraffic: 15 * TB }))), ["hetzner-forecast:1:1=warning"], "75% on the tenth is on course to pass it: the forecast, not yet the traffic alert"); assert.deepEqual(problems(stored({ outgoingTraffic: 5 * TB })), []); const warn = problems(stored({ outgoingTraffic: 17 * TB })); assert.deepEqual(keys(warn), ["hetzner-traffic:1:1=warning"]); assert.match(warn[0].message, /web has sent 17\.0 TB of its 20\.0 TB traffic this month \(85%\)/); assert.deepEqual(keys(problems(stored({ outgoingTraffic: 21 * TB }))), ["hetzner-traffic:1:1=critical"]); assert.deepEqual(problems(stored({ outgoingTraffic: null })), []); assert.deepEqual(problems(stored({ includedTraffic: null, outgoingTraffic: 99 * TB })), []); }); describe("backups", () => { const backup = (ageHours: number, serverId = 1) => ({ id: 1, type: "backup" as const, description: null, created: iso(NOW - ageHours * HOUR), sizeGb: 2, serverId }); test("enabled and recent is fine; enabled and old, or none for an old server, is a warning; not enabled is not judged", () => { const on = stored({ backupWindow: "22-02" }); assert.deepEqual(problems(on, [backup(20)]), []); assert.match(problems(on, [backup(100)])[0].message, /backups are enabled for web but the newest is 4 days old/); assert.match(problems(on, [])[0].message, /but none has been made/); assert.deepEqual(problems(on, [backup(100, 99)]).length, 1, "another server's backup does not count"); assert.deepEqual(problems(stored(), []), []); }); test("a new server has had no time to make one; the limit is the setting, and 0 turns it off; an unknown list says nothing", () => { const fresh = stored({ backupWindow: "22-02", created: iso(NOW - 5 * HOUR) }); assert.deepEqual(problems(fresh, []), []); assert.deepEqual(problems(stored({ backupWindow: "22-02" }), [backup(100)], 0), []); assert.deepEqual(problems(stored({ backupWindow: "22-02" }), [backup(100)], 200), []); assert.deepEqual(problems(stored({ backupWindow: "22-02" }), null), []); }); }); }); describe("which of the app's servers a Hetzner server is", () => { const app = [{ id: 1, name: "web-prod", ips: ["203.0.113.5", "192.168.1.5"] }, { id: 2, name: "Db", ips: [] }, { id: 3, name: "other", ips: ["10.0.0.2"] }]; test("by an address first, then by name; a server the app does not know matches none", () => { assert.equal(hz.matchAppServer({ name: "anything", ipv4: "203.0.113.5", privateIps: [] }, app)?.id, 1); assert.equal(hz.matchAppServer({ name: "x", ipv4: null, privateIps: ["10.0.0.2"] }, app)?.id, 3, "a private address counts"); assert.equal(hz.matchAppServer({ name: "db", ipv4: "198.51.100.9", privateIps: [] }, app)?.id, 2, "names are compared without regard to case"); assert.equal(hz.matchAppServer({ name: "web", ipv4: "198.51.100.9", privateIps: [] }, app), null); }); test("an address wins over a name that belongs to another server", () => { assert.equal(hz.matchAppServer({ name: "Db", ipv4: "203.0.113.5", privateIps: [] }, app)?.id, 1); }); }); describe("what is stored", () => { let id = 0; before(async () => { id = (await db.insert(integrations).values({ type: "hetzner", name: "Project", baseUrl: "https://api.hetzner.cloud", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id; }); const row = async () => (await db.select().from(hetznerState).where(eq(hetznerState.integrationId, id)))[0]; after(async () => { await hz.forgetHetzner(id); await db.delete(integrations).where(eq(integrations.id, id)); // so its servers are not matched by the tests that follow }); test("the state is replaced at each read, carrying what was seen between them; an unknown list is stored as null", async () => { const t0 = new Date(); await hz.storeRead(id, read({ volumes: null, prices: null }), t0); let r = await row(); assert.deepEqual([hz.serversOf(r).length, hz.volumesOf(r), hz.costOf(r)], [1, null, null]); await hz.storeRead(id, read({ servers: [server({ status: "off" })], volumes: [{ id: 1, name: "v", size: 10, serverId: 1, location: "fsn1", status: "available" }] }), new Date(t0.getTime() + 5 * MIN)); r = await row(); const [s] = hz.serversOf(r); assert.deepEqual([s.status, s.lastRunningAt, hz.volumesOf(r)?.length, hz.costOf(r)?.currency], ["off", t0.toISOString(), 1, "EUR"]); }); test("a read that failed is said, and what was stored is left alone", async () => { const ok = await hz.syncHetzner(id, new Date(), async () => read({ servers: [server({ name: "kept" })] })); assert.equal(ok.ok, true); const failed = await hz.syncHetzner(id, new Date(), async () => { throw new Error("connection refused"); }); assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]); assert.equal(failed.at, ok.at); assert.equal(hz.serversOf(await row())[0].name, "kept"); assert.equal(hz.isHetznerFresh((await hz.hetznerSyncStatuses())[id], Date.now()), false); }); }); describe("the health pass, the page and /metrics", () => { let hzId = 0; let app: Server; let base = ""; const ids: Record = {}; const fresh = () => setInternalFlag("hetznerSync", JSON.stringify({ [hzId]: { at: new Date().toISOString(), ok: true, error: null } })); const stale = () => setInternalFlag("hetznerSync", JSON.stringify({ [hzId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } })); before(async () => { await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]); hzId = (await db.insert(integrations).values({ type: "hetzner", name: "Prod project", baseUrl: "https://api.hetzner.cloud", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id; const mk = async (name: string, ips: string[]) => (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, ipAddresses: JSON.stringify(ips) }).returning({ id: servers.id }))[0].id; ids.web = await mk("web-agent", ["203.0.113.5"]); ids.other = await mk("elsewhere", ["192.168.9.9"]); // web: running but nearly out of traffic; db: was running and is now off; blog: rescue left on for hours. const t0 = new Date(Date.now() - 3 * HOUR); await hz.storeRead(hzId, read({ servers: [server({ id: 1, name: "web", outgoingTraffic: 19 * TB, backupWindow: "22-02" }), server({ id: 2, name: "db", ipv4: "203.0.113.6", status: "running" }), server({ id: 3, name: "blog", ipv4: "203.0.113.7", rescueEnabled: true })], volumes: [{ id: 1, name: "data", size: 100, serverId: 1, location: "fsn1", status: "available" }, { id: 2, name: "orphan", size: 20, serverId: null, location: "fsn1", status: "available" }], images: [{ id: 1, type: "snapshot", description: "before upgrade", created: iso(Date.now() - 40 * 24 * HOUR), sizeGb: 8, serverId: 1 }] }), t0); await hz.storeRead(hzId, read({ servers: [server({ id: 1, name: "web", outgoingTraffic: 19 * TB, backupWindow: "22-02" }), server({ id: 2, name: "db", ipv4: "203.0.113.6", status: "off" }), server({ id: 3, name: "blog", ipv4: "203.0.113.7", rescueEnabled: true })], volumes: [{ id: 1, name: "data", size: 100, serverId: 1, location: "fsn1", status: "available" }, { id: 2, name: "orphan", size: 20, serverId: null, location: "fsn1", status: "available" }], images: [{ id: 1, type: "snapshot", description: "before upgrade", created: iso(Date.now() - 40 * 24 * HOUR), sizeGb: 8, serverId: 1 }] }), new Date()); await fresh(); const a = express(); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: "viewer" } }; next(); }); a.use("/api/hetzner", hetznerRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/hetzner`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("the health pass judges it from what is stored, and holds it when the last read is old", async () => { let { snapshot, held } = await collectSnapshot(); assert.deepEqual(snapshot.hetzner?.map((h) => h.integrationId), [hzId]); const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true }); const hetzner = list.filter((x) => x.key.startsWith("hetzner-")).map((x) => `${x.key}=${x.severity}`).sort(); assert.deepEqual(hetzner, [`hetzner-backup:${hzId}:1=warning`, `hetzner-off:${hzId}:2=warning`, `hetzner-rescue:${hzId}:3=warning`, `hetzner-traffic:${hzId}:1=warning`].sort()); const off = list.find((x) => x.key === `hetzner-off:${hzId}:2`)!; assert.equal(off.source, `hetzner:${hzId}`); for (const k of ["off", "stuck", "rescue", "traffic", "backup"]) assert.equal(subjectOfConditionKey(`hetzner-${k}:${hzId}:2`), `integration:${hzId}`, "maintenance on the project silences its alerts"); await stale(); ({ snapshot, held } = await collectSnapshot()); assert.deepEqual([snapshot.hetzner ?? [], held.has(`hetzner:${hzId}`)], [[], true]); await fresh(); }); test("the Alerts page shows them with where to look: the backup under Backups, the traffic under Monitoring, the rest under Services", async () => { invalidateAlertsCache(); const report = await getAlerts(true); const find = (kind: string, server: number) => report.alerts.find((x) => x.id.endsWith(`hetzner-${kind}:${hzId}:${server}`))!; assert.deepEqual([find("off", 2).category, find("off", 2).source, find("off", 2).link, find("off", 2).target?.type], ["services", "Hetzner", "/hetzner", "integration"]); assert.equal(find("traffic", 1).category, "monitoring"); assert.equal(find("backup", 1).category, "backup"); assert.equal(find("rescue", 3).category, "services"); }); test("the page: each server with the app's server it is (or none), the volumes with what they hang on, the snapshots, the cost and what is wrong", async () => { const body = (await (await fetch(base)).json()) as { projects: { id: number; fresh: boolean; problems: unknown[]; state: { servers: { name: string; linked: { id: number; name: string } | null }[]; volumes: { name: string; serverName: string | null }[] | null; images: { description: string; serverName: string | null }[]; cost: { currency: string; total: number } } }[] }; const p = body.projects.find((x) => x.id === hzId)!; assert.equal(p.fresh, true); assert.deepEqual(p.state.servers.map((s) => [s.name, s.linked?.name ?? null]), [["web", "web-agent"], ["db", null], ["blog", null]]); assert.deepEqual(p.state.volumes?.map((v) => [v.name, v.serverName]), [["data", "web"], ["orphan", null]]); assert.deepEqual(p.state.images.map((i) => [i.description, i.serverName]), [["before upgrade", "web"]]); assert.equal(p.state.cost.currency, "EUR"); assert.equal(p.problems.length, 4); await stale(); const old = (await (await fetch(base)).json()) as typeof body; assert.deepEqual(old.projects.find((x) => x.id === hzId)!.problems, [], "an old reading says nothing is wrong"); await fresh(); }); test("the server's own page gets its Hetzner server, with what is wrong with it, and a server that is not one gets none", async () => { const mine = (await (await fetch(`${base}/for-server/${ids.web}`)).json()) as { match: { integrationName: string; fresh: boolean; server: { name: string; includedTraffic: number }; problems: { key: string }[]; snapshots: number; newestBackup: string | null } | null }; assert.deepEqual([mine.match?.integrationName, mine.match?.server.name, mine.match?.snapshots, mine.match?.newestBackup], ["Prod project", "web", 1, null]); assert.deepEqual(mine.match?.problems.map((p) => p.key).sort(), [`hetzner-backup:${hzId}:1`, `hetzner-traffic:${hzId}:1`]); const none = (await (await fetch(`${base}/for-server/${ids.other}`)).json()) as { match: unknown }; assert.equal(none.match, null); assert.equal((await fetch(`${base}/for-server/999999`)).status, 404); assert.equal((await fetch(`${base}/for-server/abc`)).status, 400); }); test("/metrics has whether each server runs, its traffic against the quota and the project's cost — only while the reading is recent", async () => { let text = await buildMetrics(new Date()); assert.match(text, /homelab_hetzner_server_running\{project="Prod project",server="web"\} 1/); assert.match(text, /homelab_hetzner_server_running\{project="Prod project",server="db"\} 0/); assert.match(text, /homelab_hetzner_traffic_included_bytes\{project="Prod project",server="web"\}/); assert.match(text, /homelab_hetzner_monthly_cost\{[^}]*currency="EUR"[^}]*\} \d/); await stale(); text = await buildMetrics(new Date()); assert.doesNotMatch(text, /homelab_hetzner/); await fresh(); }); test("taking the integration away takes what was read with it", async () => { await hz.forgetHetzner(hzId); assert.equal((await hz.hetznerSyncStatuses())[hzId], undefined); await db.delete(integrations).where(eq(integrations.id, hzId)); assert.deepEqual(await db.select().from(hetznerState).where(eq(hetznerState.integrationId, hzId)), []); }); });