/** * Hetzner Cloud, the second round: load balancers (and the health of their targets), floating IPs, certificates (and when they end), their share of the cost, and the traffic forecast * ("at this rate the server passes its quota on the 24th"). What the API says, what is stored, what is wrong, the page's data, the health pass and /metrics. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { createServer as createHttpServer, type Server } from "node:http"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-hetzner2-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, integrations } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const adapter = await import("../src/integrations/hetzner/adapter.js"); const hz = await import("../src/services/hetznerSync.js"); const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js"); const { subjectOfConditionKey } = await import("../src/services/maintenance.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { hetznerRouter } = await import("../src/routes/hetzner.js"); const { buildMetrics } = await import("../src/services/metrics.js"); const { setInternalFlag } = await import("../src/services/settingsStore.js"); await runMigrations(); const MIN = 60_000; const HOUR = 60 * MIN; const DAY = 24 * HOUR; const TB = 1024 ** 4; const iso = (ms: number) => new Date(ms).toISOString(); type Server_ = import("../src/integrations/hetzner/adapter.js").HetznerServer; type Read = import("../src/integrations/hetzner/adapter.js").HetznerRead; type Lb = import("../src/integrations/hetzner/adapter.js").HetznerLoadBalancer; type Cert = import("../src/integrations/hetzner/adapter.js").HetznerCertificate; type Fip = import("../src/integrations/hetzner/adapter.js").HetznerFloatingIp; const server = (extra: Partial = {}): Server_ => ({ id: 1, name: "web", status: "running", created: iso(Date.now() - 30 * DAY), type: "cx22", cores: 2, memoryGb: 4, diskGb: 40, location: "fsn1", city: "Falkenstein", os: "Ubuntu 24.04", ipv4: "203.0.113.5", ipv6: null, privateIps: [], locked: false, rescueEnabled: false, backupWindow: null, includedTraffic: 20 * TB, outgoingTraffic: 1 * TB, ingoingTraffic: 2 * TB, monthlyPrice: 4, ...extra }); const lb = (extra: Partial = {}): Lb => ({ id: 10, name: "front", type: "lb11", location: "fsn1", ipv4: "203.0.113.50", ipv6: null, algorithm: "round_robin", services: 1, targets: [{ kind: "server", serverId: 1, label: null, health: "healthy" }], includedTraffic: 20 * TB, outgoingTraffic: 0, monthlyPrice: 6, ...extra }); const cert = (extra: Partial = {}): Cert => ({ id: 20, name: "site", type: "managed", domains: ["example.se", "www.example.se"], notValidAfter: iso(Date.now() + 60 * DAY), issuance: "completed", renewal: "completed", error: null, usedBy: 1, ...extra }); const fip = (extra: Partial = {}): Fip => ({ id: 30, name: "vip", ip: "203.0.113.60", type: "ipv4", serverId: 1, location: "fsn1", description: null, ...extra }); const PRICES = { currency: "EUR", imagePerGb: 0.012, volumePerGb: 0.05, backupPercent: 20, primaryIps: [], floatingIps: [{ type: "ipv4", location: "fsn1", monthly: 3.5 }] }; describe("what Hetzner says of load balancers, floating IPs and certificates", () => { test("load balancers: type and place (and its price there), address, services — and each target's health: unhealthy as soon as one port is", () => { const [l] = adapter.parseLoadBalancers({ load_balancers: [ { id: 10, name: "front", public_net: { ipv4: { ip: "203.0.113.50" }, ipv6: { ip: "2001:db8::1" } }, load_balancer_type: { name: "lb11", prices: [{ location: "nbg1", price_monthly: { gross: "5.0" } }, { location: "fsn1", price_monthly: { gross: "6.4" } }] }, location: { name: "fsn1" }, algorithm: { type: "round_robin" }, services: [{ listen_port: 80 }, { listen_port: 443 }], included_traffic: 21_990_232_555_520, outgoing_traffic: 1000, targets: [ { type: "server", server: { id: 1 }, health_status: [{ listen_port: 80, status: "healthy" }, { listen_port: 443, status: "healthy" }] }, { type: "server", server: { id: 2 }, health_status: [{ listen_port: 80, status: "healthy" }, { listen_port: 443, status: "unhealthy" }] }, { type: "server", server: { id: 3 }, health_status: [] }, { type: "ip", ip: { ip: "198.51.100.7" }, health_status: [{ listen_port: 80, status: "unhealthy" }] }, ], }, ], }); assert.deepEqual([l.name, l.type, l.location, l.ipv4, l.algorithm, l.services, l.monthlyPrice, l.includedTraffic], ["front", "lb11", "fsn1", "203.0.113.50", "round_robin", 2, 6.4, 21_990_232_555_520]); assert.deepEqual(l.targets.map((t) => [t.kind, t.serverId ?? t.label, t.health]), [["server", 1, "healthy"], ["server", 2, "unhealthy"], ["server", 3, "unknown"], ["ip", "198.51.100.7", "unhealthy"]]); }); test("a label selector is the servers it found (each with its own health), or itself when it found none", () => { const [l] = adapter.parseLoadBalancers({ load_balancers: [ { id: 1, name: "x", targets: [ { type: "label_selector", label_selector: { selector: "env=prod" }, targets: [{ type: "server", server: { id: 4 }, health_status: [{ status: "healthy" }] }, { type: "server", server: { id: 5 }, health_status: [{ status: "unhealthy" }] }] }, { type: "label_selector", label_selector: { selector: "env=none" }, targets: [] }, ], }, ], }); assert.deepEqual(l.targets.map((t) => [t.serverId, t.label, t.health]), [[4, "env=prod", "healthy"], [5, "env=prod", "unhealthy"], [null, "env=none", "unknown"]]); }); test("without an id or a name, or with nothing to read, a load balancer is skipped and the fields are null", () => { assert.deepEqual(adapter.parseLoadBalancers({ load_balancers: [{ id: 1 }, { name: "x" }, "junk"] }), []); const [bare] = adapter.parseLoadBalancers({ load_balancers: [{ id: 2, name: "bare" }] }); assert.deepEqual([bare.type, bare.ipv4, bare.monthlyPrice, bare.targets, bare.services], [null, null, null, [], 0]); assert.deepEqual(adapter.parseLoadBalancers(null), []); }); test("floating IPs: the address, what it points at (nothing is none) and where it lives", () => { const list = adapter.parseFloatingIps({ floating_ips: [{ id: 1, name: "vip", ip: "203.0.113.60", type: "ipv4", server: 5, home_location: { name: "fsn1" }, description: "web" }, { id: 2, ip: "203.0.113.61", type: "ipv4", server: null }, { id: 3, ip: "x", type: "weird" }, { id: 4 }] }); assert.deepEqual(list.map((f) => [f.ip, f.serverId, f.location, f.name]), [["203.0.113.60", 5, "fsn1", "vip"], ["203.0.113.61", null, null, null]]); }); test("certificates: the domains, when it ends, how its issuing and renewal went, and how many things use it", () => { const list = adapter.parseCertificates({ certificates: [ { id: 1, name: "site", type: "managed", domain_names: ["example.se", "www.example.se"], not_valid_after: "2026-09-01T10:00:00+00:00", status: { issuance: "completed", renewal: "scheduled" }, used_by: [{ id: 10, type: "load_balancer" }] }, { id: 2, name: "new", type: "managed", domain_names: ["a.se"], not_valid_after: null, status: { issuance: "failed", error: { code: "x", message: "DNS did not answer" } }, used_by: [] }, { id: 3 }, ], }); assert.deepEqual(list.map((c) => [c.name, c.type, c.domains.length, c.notValidAfter, c.issuance, c.error, c.usedBy]), [["site", "managed", 2, "2026-09-01T10:00:00.000Z", "completed", null, 1], ["new", "managed", 1, null, "failed", "DNS did not answer", 0]]); }); test("the price list: a floating IP's price for each kind and place — or the one price an older list has", () => { const p = adapter.parsePrices({ pricing: { currency: "EUR", floating_ips: [{ type: "ipv4", prices: [{ location: "fsn1", price_monthly: { gross: "3.5700" } }] }] } })!; assert.deepEqual(p.floatingIps, [{ type: "ipv4", location: "fsn1", monthly: 3.57 }]); const old = adapter.parsePrices({ pricing: { currency: "EUR", floating_ip: { price_monthly: { gross: "3.1" } } } })!; assert.deepEqual(old.floatingIps.map((f) => [f.type, f.location, f.monthly]), [["ipv4", "*", 3.1], ["ipv6", "*", 3.1]]); }); describe("the adapter against a Hetzner that answers", () => { let fake: Server; let url = ""; let lbStatus = 200; before(async () => { fake = createHttpServer((req, res) => { const u = new URL(req.url ?? "", "http://x"); const send = (code: number, body: unknown) => { res.writeHead(code, { "content-type": "application/json" }); res.end(JSON.stringify(body)); }; const page = { meta: { pagination: { page: 1, next_page: null } } }; if (u.pathname === "/v1/servers") return send(200, { servers: [{ id: 1, name: "web" }], ...page }); if (u.pathname === "/v1/load_balancers") return lbStatus === 200 ? send(200, { load_balancers: [{ id: 10, name: "front", targets: [] }], ...page }) : send(lbStatus, {}); if (u.pathname === "/v1/floating_ips") return send(200, { floating_ips: [{ id: 30, ip: "203.0.113.60", type: "ipv4", server: null }], ...page }); if (u.pathname === "/v1/certificates") return send(200, { certificates: [{ id: 20, name: "site", used_by: [] }], ...page }); if (u.pathname === "/v1/pricing") return send(200, { pricing: { currency: "EUR" } }); send(200, { volumes: [], images: [], primary_ips: [], ...page }); }); await new Promise((resolve) => fake.listen(0, "127.0.0.1", resolve)); url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}/v1`; }); after(() => fake.close()); test("they are read with the rest; a list that cannot be asked is unknown (null), not empty", async () => { const r = await adapter.createHetznerAdapter({ token: "t", apiUrl: url }).read(); assert.deepEqual([r.loadBalancers?.length, r.floatingIps?.[0].serverId, r.certificates?.[0].name], [1, null, "site"]); lbStatus = 500; const r2 = await adapter.createHetznerAdapter({ token: "t", apiUrl: url }).read(); assert.deepEqual([r2.loadBalancers, r2.floatingIps?.length, r2.servers.length], [null, 1, 1]); lbStatus = 200; }); }); }); describe("the cost with load balancers and floating IPs", () => { test("a load balancer is its type's price, a floating IP its kind's and place's; one without a price is counted as unpriced", () => { const base = { servers: [server({ monthlyPrice: 10 })], volumes: [], images: [], primaryIps: [], prices: PRICES }; const c = hz.estimateCost({ ...base, loadBalancers: [lb({ monthlyPrice: 6 }), lb({ id: 11, monthlyPrice: null })], floatingIps: [fip(), fip({ id: 31, location: "hel1" })] })!; assert.deepEqual([c.loadBalancers, c.floatingIps, c.total, c.unpriced], [6, 3.5, 19.5, 2]); }); test("a list that could not be read adds nothing", () => { const c = hz.estimateCost({ servers: [server({ monthlyPrice: 10 })], volumes: [], images: [], primaryIps: [], prices: PRICES, loadBalancers: null, floatingIps: null })!; assert.deepEqual([c.loadBalancers, c.floatingIps, c.total], [0, 0, 10]); }); test("an older price list's one price counts for every place", () => { const c = hz.estimateCost({ servers: [], volumes: [], images: [], primaryIps: [], prices: { ...PRICES, floatingIps: [{ type: "ipv4", location: "*", monthly: 3 }] }, floatingIps: [fip({ location: "ash" })] })!; assert.equal(c.floatingIps, 3); }); }); describe("the traffic history", () => { const t0 = new Date("2026-06-10T00:00:00Z"); test("one sample about every six hours, not more often; two weeks are kept", () => { let h = hz.carryTraffic(1 * TB, [], t0); h = hz.carryTraffic(1.1 * TB, h, new Date(t0.getTime() + HOUR)); assert.equal(h.length, 1, "a read an hour later adds nothing"); h = hz.carryTraffic(1.2 * TB, h, new Date(t0.getTime() + 7 * HOUR)); assert.equal(h.length, 2); h = hz.carryTraffic(5 * TB, h, new Date(t0.getTime() + 15 * DAY)); assert.deepEqual(h.map((x) => x.bytes), [5 * TB], "what is over two weeks old is dropped"); }); test("a counter that goes down is a new month: the history starts over", () => { let h = hz.carryTraffic(18 * TB, [], t0); h = hz.carryTraffic(0.2 * TB, h, new Date(t0.getTime() + 7 * HOUR)); assert.deepEqual(h.map((x) => x.bytes), [0.2 * TB]); }); test("a server that said nothing of its traffic keeps what it had", () => { const h = [{ at: t0.toISOString(), bytes: 1 }]; assert.deepEqual(hz.carryTraffic(null, h, new Date(t0.getTime() + DAY)), h); }); test("carryTimes keeps it from read to read", () => { const a = hz.carryTimes([server({ outgoingTraffic: 1 * TB })], [], t0); const b = hz.carryTimes([server({ outgoingTraffic: 2 * TB })], a, new Date(t0.getTime() + 7 * HOUR)); assert.deepEqual(b[0].trafficHistory?.map((x) => x.bytes), [1 * TB, 2 * TB]); }); }); describe("the traffic forecast", () => { const NOW = Date.parse("2026-06-11T00:00:00Z"); // ten days into a month of thirty const sample = (daysAgo: number, bytes: number) => ({ at: iso(NOW - daysAgo * DAY), bytes }); const s = (used: number, history: { at: string; bytes: number }[] = [], included = 20 * TB) => ({ outgoingTraffic: used, includedTraffic: included, trafficHistory: history }); test("by the month's average when there is no recent history: 10 TB in ten days is 1 TB a day, and 30 TB at the month's end — the quota goes on the 21st", () => { const f = hz.trafficForecast(s(10 * TB), NOW)!; assert.equal(f.basis, "month"); assert.equal(Math.round(f.perDay / TB), 1); assert.equal(Math.round(f.projectedMonth / TB), 30); assert.equal(f.exhaustedAt && hz.dayText(f.exhaustedAt), "21 June"); }); test("by the last week's rate when there is some: a rate that has gone up counts", () => { // 4 TB by day 4 (seven days ago is before the month's start, so the sample at day 4 is the oldest in the window), then 10 TB now: 1 TB a day. const f = hz.trafficForecast(s(10 * TB, [sample(6, 4 * TB), sample(3, 6 * TB)]), NOW)!; assert.equal(f.basis, "recent"); assert.equal(Math.round(f.perDay / TB), 1); // The last three days were quicker than the month: 6 -> 10 TB over 3 days is 1.33 TB a day, and that is what the oldest sample in a three-day window gives. const fast = hz.trafficForecast(s(10 * TB, [sample(3, 6 * TB)]), NOW)!; assert.equal(fast.basis, "recent"); assert.ok(fast.perDay > 1.2 * TB && fast.perDay < 1.5 * TB); }); test("a forecast that stays under the quota has no date; a quota already passed has none either", () => { const calm = hz.trafficForecast(s(2 * TB), NOW)!; assert.deepEqual([calm.exhaustedAt, Math.round(calm.projectedMonth / TB)], [null, 6]); assert.equal(hz.trafficForecast(s(21 * TB), NOW)!.exhaustedAt, null); }); test("too little to say: the first days of a month, a server with no quota or no traffic, a counter that says nothing", () => { const early = Date.parse("2026-06-02T00:00:00Z"); assert.equal(hz.trafficForecast(s(1 * TB), early), null); assert.equal(hz.trafficForecast(s(0), NOW), null); assert.equal(hz.trafficForecast({ outgoingTraffic: null, includedTraffic: 20 * TB }, NOW), null); assert.equal(hz.trafficForecast({ outgoingTraffic: 5 * TB, includedTraffic: null }, NOW), null); }); test("samples from last month, or higher than now, are not used", () => { const f = hz.trafficForecast(s(10 * TB, [sample(20, 19 * TB), sample(5, 30 * TB)]), NOW)!; assert.equal(f.basis, "month"); }); }); describe("what is wrong, the second round", () => { const NOW = Date.parse("2026-06-11T00:00:00Z"); const stored = (extra: Partial = {}, hist: { at: string; bytes: number }[] = []) => ({ ...server(extra), lastRunningAt: iso(NOW - HOUR), statusSince: iso(NOW - HOUR), rescueSince: null, trafficHistory: hist }); const keys = (list: ReturnType) => list.map((p) => `${p.key}=${p.severity}`); const problems = (extras: import("../src/services/hetznerSync.js").HetznerExtras, servers = [stored()]) => hz.hetznerProblems(1, "Prod", servers, [], NOW, 48, extras); describe("load balancers", () => { test("healthy says nothing; one unhealthy target is a warning naming it; all of them is critical; an unknown is not unhealthy", () => { assert.deepEqual(problems({ loadBalancers: [lb()] }), []); const one = problems({ loadBalancers: [lb({ targets: [{ kind: "server", serverId: 1, label: null, health: "healthy" }, { kind: "server", serverId: 2, label: null, health: "unhealthy" }] })] }, [stored(), stored({ id: 2, name: "api" })]); assert.deepEqual(keys(one), ["hetzner-lb:1:10=warning"]); assert.match(one[0].message, /Prod: load balancer front has 1 of 2 targets unhealthy \(api\)$/); const all = problems({ loadBalancers: [lb({ targets: [{ kind: "server", serverId: 1, label: null, health: "unhealthy" }] })] }); assert.deepEqual(keys(all), ["hetzner-lb:1:10=critical"]); assert.match(all[0].message, /nothing healthy to send to/); assert.deepEqual(problems({ loadBalancers: [lb({ targets: [{ kind: "server", serverId: 1, label: null, health: "unknown" }] })] }), []); assert.deepEqual(problems({ loadBalancers: null }), []); }); }); describe("certificates", () => { test("far from the end is fine; within two weeks is a warning, within three days critical, past the end critical; the message says who must renew it", () => { assert.deepEqual(problems({ certificates: [cert()] }), []); const soon = problems({ certificates: [cert({ notValidAfter: iso(NOW + 10 * DAY) })] }); assert.deepEqual(keys(soon), ["hetzner-cert:1:20=warning"]); assert.match(soon[0].message, /certificate site \(example\.se, www\.example\.se\) ends in 10 days — Hetzner should have renewed it/); assert.deepEqual(keys(problems({ certificates: [cert({ notValidAfter: iso(NOW + 2 * DAY), type: "uploaded" })] })), ["hetzner-cert:1:20=critical"]); assert.match(problems({ certificates: [cert({ notValidAfter: iso(NOW + 2 * DAY), type: "uploaded" })] })[0].message, /uploaded, so it has to be renewed by hand/); const gone = problems({ certificates: [cert({ notValidAfter: iso(NOW - 3 * DAY) })] }); assert.deepEqual(keys(gone), ["hetzner-cert:1:20=critical"]); assert.match(gone[0].message, /ended 3 days ago/); }); test("a managed one that could not be issued or renewed says why; one nothing uses is not news; an unknown end or an unread list says nothing", () => { const failed = problems({ certificates: [cert({ renewal: "failed", error: "DNS did not answer", notValidAfter: iso(NOW + 59 * DAY + HOUR) })] }); assert.deepEqual(keys(failed), ["hetzner-cert:1:20=warning"]); assert.match(failed[0].message, /could not be renewed by Hetzner — DNS did not answer \(it is valid for 59 more days\)/); assert.deepEqual(problems({ certificates: [cert({ notValidAfter: iso(NOW + DAY), usedBy: 0 })] }), []); assert.deepEqual(problems({ certificates: [cert({ notValidAfter: null })] }), []); assert.deepEqual(problems({ certificates: null }), []); }); }); describe("floating IPs", () => { test("one that points at a server is fine; one that points at none is a warning (it is billed)", () => { assert.deepEqual(problems({ floatingIps: [fip()] }), []); const free = problems({ floatingIps: [fip({ serverId: null })] }); assert.deepEqual(keys(free), ["hetzner-fip:1:30=warning"]); assert.match(free[0].message, /floating IP 203\.0\.113\.60 \(vip\) points at no server — it is billed all the same/); }); }); describe("the traffic forecast", () => { test("on course to pass the quota is a warning with the date, the rate and what it is made from; the traffic alert takes over from 80%", () => { const p = problems({}, [stored({ outgoingTraffic: 10 * TB })]); assert.deepEqual(keys(p), ["hetzner-forecast:1:1=warning"]); assert.match(p[0].message, /Prod: at this rate web will pass its 20\.0 TB traffic quota on 21 June — it has sent 10\.0 TB so far, about 1\.0 TB a day \(the month's average\)/); assert.deepEqual(keys(problems({}, [stored({ outgoingTraffic: 17 * TB })])), ["hetzner-traffic:1:1=warning"], "no forecast beside the traffic alert"); assert.deepEqual(problems({}, [stored({ outgoingTraffic: 3 * TB })]), []); }); }); }); describe("the health pass, the page and /metrics", () => { let id = 0; let app: Server; let base = ""; const fresh = () => setInternalFlag("hetznerSync", JSON.stringify({ [id]: { at: new Date().toISOString(), ok: true, error: null } })); const stale = () => setInternalFlag("hetznerSync", JSON.stringify({ [id]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } })); const sick = lb({ id: 11, name: "api-lb", targets: [{ kind: "server", serverId: 2, label: null, health: "unhealthy" }] }); before(async () => { await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]); id = (await db.insert(integrations).values({ type: "hetzner", name: "Prod project", baseUrl: "https://api.hetzner.cloud", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id; // web has used 11 TB of 20 and the month is a few days old at least: a forecast either way is not what is tested here, so give it nothing. const read: Read = { servers: [server({ id: 1, name: "web", outgoingTraffic: 1 * TB }), server({ id: 2, name: "api", ipv4: "203.0.113.9" })], volumes: [], images: [], primaryIps: [], prices: PRICES, loadBalancers: [lb(), sick], floatingIps: [fip(), fip({ id: 31, ip: "203.0.113.61", serverId: null })], certificates: [cert({ notValidAfter: iso(Date.now() + 5 * DAY) }), cert({ id: 21, name: "unused", notValidAfter: iso(Date.now() + DAY), usedBy: 0 })], }; await hz.storeRead(id, read, new Date()); await fresh(); const a = express(); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: "viewer" } }; next(); }); a.use("/api/hetzner", hetznerRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/hetzner`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("they are stored (an unread list as null) and come back", async () => { const body = (await (await fetch(base)).json()) as { projects: { id: number; state: { loadBalancers: { name: string; targets: { serverName: string | null }[] }[] | null; floatingIps: { ip: string; serverName: string | null }[]; certificates: { name: string }[]; cost: { loadBalancers: number; floatingIps: number }; servers: { name: string; trafficHistory?: unknown; forecast: unknown }[] } }[] }; const p = body.projects.find((x) => x.id === id)!; assert.deepEqual(p.state.loadBalancers?.map((l) => [l.name, l.targets[0].serverName]), [["front", "web"], ["api-lb", "api"]]); assert.deepEqual(p.state.floatingIps.map((f) => [f.ip, f.serverName]), [["203.0.113.60", "web"], ["203.0.113.61", null]]); assert.deepEqual(p.state.certificates.map((c) => c.name), ["site", "unused"]); assert.deepEqual([p.state.cost.loadBalancers, p.state.cost.floatingIps], [12, 7]); assert.ok(p.state.servers.every((s) => s.trafficHistory === undefined), "the history is not sent to the page"); }); test("the health pass judges them, and maintenance on the integration silences them", async () => { const { snapshot } = await collectSnapshot(); const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true }); const mine = list.filter((x) => /^hetzner-(lb|cert|fip):/.test(x.key)).map((x) => `${x.key}=${x.severity}`).sort(); assert.deepEqual(mine, [`hetzner-cert:${id}:20=warning`, `hetzner-fip:${id}:31=warning`, `hetzner-lb:${id}:11=critical`].sort(), "the certificate nothing uses is left out"); for (const k of ["lb", "cert", "fip", "forecast"]) assert.equal(subjectOfConditionKey(`hetzner-${k}:${id}:5`), `integration:${id}`); }); test("the Alerts page says where to look", async () => { invalidateAlertsCache(); const report = await getAlerts(true); const find = (kind: string, n: number) => report.alerts.find((x) => x.id.endsWith(`hetzner-${kind}:${id}:${n}`))!; assert.deepEqual([find("lb", 11).category, find("cert", 20).category, find("fip", 31).category, find("lb", 11).link], ["services", "services", "services", "/hetzner"]); }); test("/metrics has each load balancer's unhealthy targets and each certificate's end — only while the reading is recent", async () => { let text = await buildMetrics(new Date()); assert.match(text, /homelab_hetzner_load_balancer_unhealthy_targets\{project="Prod project",load_balancer="api-lb"\} 1/); assert.match(text, /homelab_hetzner_load_balancer_unhealthy_targets\{project="Prod project",load_balancer="front"\} 0/); assert.match(text, /homelab_hetzner_certificate_expiry_timestamp_seconds\{project="Prod project",certificate="site"\} \d/); await stale(); text = await buildMetrics(new Date()); assert.doesNotMatch(text, /homelab_hetzner/); await fresh(); }); });