One integration per project (read-only API token, read every 5 minutes). Alerts for a server that was running and is off, stuck statuses, traffic at 80%/100% of the monthly quota, rescue mode left on, and late backups. The Hetzner page lists servers, volumes, snapshots and a cost estimate; a server the app knows by its agent gets a Hetzner card on its own page. /metrics and docs updated. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
439 lines
30 KiB
TypeScript
439 lines
30 KiB
TypeScript
/**
|
|
* Hetzner Cloud: reading what its API says (paged, read-only), what is stored and carried between reads, the cost estimate, what is wrong, how a Hetzner server is matched to one the app
|
|
* knows, the health pass, the alerts, the page's routes and /metrics.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { createServer as createHttpServer, type Server } from "node:http";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { AddressInfo } from "node:net";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-hetzner-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const express = (await import("express")).default;
|
|
const { db } = await import("../src/db/client.js");
|
|
const { users, integrations, servers, hetznerState } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const adapter = await import("../src/integrations/hetzner/adapter.js");
|
|
const hz = await import("../src/services/hetznerSync.js");
|
|
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
|
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
|
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
|
const { hetznerRouter } = await import("../src/routes/hetzner.js");
|
|
const { buildMetrics } = await import("../src/services/metrics.js");
|
|
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
|
const { eq } = await import("drizzle-orm");
|
|
|
|
await runMigrations();
|
|
|
|
const MIN = 60_000;
|
|
const HOUR = 60 * MIN;
|
|
const TB = 1024 ** 4;
|
|
const iso = (ms: number) => new Date(ms).toISOString();
|
|
|
|
type Read = import("../src/integrations/hetzner/adapter.js").HetznerRead;
|
|
type Server_ = import("../src/integrations/hetzner/adapter.js").HetznerServer;
|
|
const server = (extra: Partial<Server_> = {}): Server_ => ({ id: 1, name: "web", status: "running", created: iso(Date.now() - 30 * 24 * HOUR), type: "cx22", cores: 2, memoryGb: 4, diskGb: 40, location: "fsn1", city: "Falkenstein", os: "Ubuntu 24.04", ipv4: "203.0.113.5", ipv6: "2001:db8::/64", privateIps: ["10.0.0.2"], locked: false, rescueEnabled: false, backupWindow: null, includedTraffic: 20 * TB, outgoingTraffic: 1 * TB, ingoingTraffic: 2 * TB, monthlyPrice: 4, ...extra });
|
|
const PRICES = { currency: "EUR", imagePerGb: 0.012, volumePerGb: 0.05, backupPercent: 20, primaryIps: [{ type: "ipv4", location: "fsn1", monthly: 0.5 }] };
|
|
const read = (extra: Partial<Read> = {}): Read => ({ servers: [server()], volumes: [], images: [], primaryIps: [], prices: PRICES, ...extra });
|
|
|
|
// What Hetzner sends (the OpenAPI file's shapes; prices are strings).
|
|
const apiServer = (id: number, name: string, extra: Record<string, unknown> = {}) => ({
|
|
id,
|
|
name,
|
|
status: "running",
|
|
created: "2026-01-02T03:04:05+00:00",
|
|
public_net: { ipv4: { id: 1, ip: `203.0.113.${id}`, blocked: false, dns_ptr: "x" }, ipv6: { id: 2, ip: "2001:db8::/64", blocked: false, dns_ptr: [] }, floating_ips: [], firewalls: [] },
|
|
private_net: [{ network: 7, ip: `10.0.0.${id}`, alias_ips: [], mac_address: "86:00:00:aa:bb:cc" }],
|
|
server_type: { id: 22, name: "cx22", cores: 2, memory: 4, disk: 40, prices: [{ location: "nbg1", price_monthly: { net: "3.0000", gross: "3.5700" } }, { location: "fsn1", price_monthly: { net: "3.2900", gross: "3.9151" } }] },
|
|
location: { id: 1, name: "fsn1", city: "Falkenstein", country: "DE" },
|
|
image: { id: 5, type: "system", name: "ubuntu-24.04", description: "Ubuntu 24.04" },
|
|
rescue_enabled: false,
|
|
locked: false,
|
|
backup_window: null,
|
|
outgoing_traffic: 1_000_000_000,
|
|
ingoing_traffic: 2_000_000_000,
|
|
included_traffic: 21_990_232_555_520,
|
|
...extra,
|
|
});
|
|
|
|
describe("what Hetzner's API says", () => {
|
|
test("servers: status, type, where, the addresses, the traffic, and the price of its type at its location (including VAT)", () => {
|
|
const [s] = adapter.parseServers({ servers: [apiServer(5, "web", { backup_window: "22-02", rescue_enabled: true })] });
|
|
assert.deepEqual([s.id, s.name, s.status, s.type, s.location, s.city, s.os], [5, "web", "running", "cx22", "fsn1", "Falkenstein", "Ubuntu 24.04"]);
|
|
assert.deepEqual([s.ipv4, s.privateIps, s.rescueEnabled, s.backupWindow, s.includedTraffic, s.outgoingTraffic], ["203.0.113.5", ["10.0.0.5"], true, "22-02", 21_990_232_555_520, 1_000_000_000]);
|
|
assert.equal(s.monthlyPrice, 3.9151, "the fsn1 price, not the nbg1 one");
|
|
});
|
|
|
|
test("a server without an id or a name is skipped, and missing parts are null, not guessed", () => {
|
|
const list = adapter.parseServers({ servers: [{ name: "x" }, { id: 2 }, { id: 3, name: "bare" }, "junk"] });
|
|
assert.deepEqual(list.map((s) => s.name), ["bare"]);
|
|
assert.deepEqual([list[0].status, list[0].ipv4, list[0].includedTraffic, list[0].monthlyPrice, list[0].backupWindow, list[0].privateIps], ["unknown", null, null, null, null, []]);
|
|
assert.deepEqual(adapter.parseServers(null), []);
|
|
});
|
|
|
|
test("volumes: the size, and the server it is attached to (none is none)", () => {
|
|
const v = adapter.parseVolumes({ volumes: [{ id: 1, name: "data", size: 100, server: 5, location: { name: "fsn1" }, status: "available" }, { id: 2, name: "old", size: 50, server: null }, { id: 3 }] });
|
|
assert.deepEqual(v.map((x) => [x.name, x.size, x.serverId, x.location]), [["data", 100, 5, "fsn1"], ["old", 50, null, null]]);
|
|
});
|
|
|
|
test("snapshots and backups: a backup belongs to bound_to, a snapshot says created_from; system images are not kept", () => {
|
|
const list = adapter.parseImages({
|
|
images: [
|
|
{ id: 1, type: "backup", description: "auto", created: "2026-05-01T00:00:00+00:00", image_size: 3.5, bound_to: 5 },
|
|
{ id: 2, type: "snapshot", description: "before upgrade", created: "2026-04-01T00:00:00+00:00", image_size: 8, created_from: { id: 5, name: "web" } },
|
|
{ id: 3, type: "system", description: "Ubuntu" },
|
|
],
|
|
});
|
|
assert.deepEqual(list.map((i) => [i.type, i.serverId, i.sizeGb]), [["backup", 5, 3.5], ["snapshot", 5, 8]]);
|
|
});
|
|
|
|
test("the price list: the per-GB prices, the backup percentage and the primary IP prices; something that is not a price list is not read", () => {
|
|
const p = adapter.parsePrices({ pricing: { currency: "EUR", image: { price_per_gb_month: { gross: "0.0119" } }, volume: { price_per_gb_month: { gross: "0.0476" } }, server_backup: { percentage: "20" }, primary_ips: [{ type: "ipv4", prices: [{ location: "fsn1", price_monthly: { gross: "0.5950" } }] }] } })!;
|
|
assert.deepEqual([p.currency, p.imagePerGb, p.volumePerGb, p.backupPercent, p.primaryIps], ["EUR", 0.0119, 0.0476, 20, [{ type: "ipv4", location: "fsn1", monthly: 0.595 }]]);
|
|
assert.equal(adapter.parsePrices({ message: "no" }), null);
|
|
});
|
|
|
|
test("primary IPs: whether one belongs to a server", () => {
|
|
const ips = adapter.parsePrimaryIps({ primary_ips: [{ id: 1, ip: "203.0.113.5", type: "ipv4", location: { name: "fsn1" }, assignee_id: 5 }, { id: 2, ip: "203.0.113.6", type: "ipv4", assignee_id: null }, { id: 3, ip: "x", type: "weird" }] });
|
|
assert.deepEqual(ips.map((i) => [i.ip, i.assigned]), [["203.0.113.5", true], ["203.0.113.6", false]]);
|
|
});
|
|
|
|
test("the next page is the one the meta says; the last has none", () => {
|
|
assert.equal(adapter.nextPage({ meta: { pagination: { page: 1, next_page: 2 } } }), 2);
|
|
assert.equal(adapter.nextPage({ meta: { pagination: { page: 2, next_page: null } } }), null);
|
|
assert.equal(adapter.nextPage({}), null);
|
|
});
|
|
|
|
describe("the adapter against a Hetzner that answers", () => {
|
|
let fake: Server;
|
|
let url = "";
|
|
let tokenOk = true;
|
|
let volumesStatus = 200;
|
|
let rateLimited = false;
|
|
const seen: { method: string | undefined; path: string; auth: string | undefined }[] = [];
|
|
before(async () => {
|
|
fake = createHttpServer((req, res) => {
|
|
const u = new URL(req.url ?? "", "http://x");
|
|
seen.push({ method: req.method, path: u.pathname + u.search, auth: req.headers.authorization });
|
|
const send = (code: number, body: unknown) => {
|
|
res.writeHead(code, { "content-type": "application/json" });
|
|
res.end(JSON.stringify(body));
|
|
};
|
|
if (!tokenOk) return send(401, { error: { code: "unauthorized" } });
|
|
if (rateLimited) return send(429, { error: { code: "rate_limit_exceeded" } });
|
|
const page = Number(u.searchParams.get("page") ?? 1);
|
|
if (u.pathname === "/v1/servers") return send(200, { servers: [apiServer(page, `s${page}`)], meta: { pagination: { page, next_page: page < 2 ? page + 1 : null } } });
|
|
if (u.pathname === "/v1/volumes") return volumesStatus === 200 ? send(200, { volumes: [{ id: 1, name: "data", size: 10, server: 1, status: "available" }], meta: { pagination: { page: 1, next_page: null } } }) : send(volumesStatus, {});
|
|
if (u.pathname === "/v1/images") return send(200, { images: u.searchParams.get("type") === "backup" ? [{ id: 9, type: "backup", created: "2026-05-01T00:00:00+00:00", image_size: 2, bound_to: 1 }] : [{ id: 8, type: "snapshot", created: "2026-04-01T00:00:00+00:00", image_size: 5, created_from: { id: 1 } }], meta: { pagination: { page: 1, next_page: null } } });
|
|
if (u.pathname === "/v1/primary_ips") return send(200, { primary_ips: [], meta: { pagination: { page: 1, next_page: null } } });
|
|
if (u.pathname === "/v1/pricing") return send(200, { pricing: { currency: "EUR", image: { price_per_gb_month: { gross: "0.0119" } }, volume: { price_per_gb_month: { gross: "0.0476" } }, server_backup: { percentage: "20" }, primary_ips: [] } });
|
|
send(404, {});
|
|
});
|
|
await new Promise<void>((resolve) => fake.listen(0, "127.0.0.1", resolve));
|
|
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}/v1`;
|
|
});
|
|
after(() => fake.close());
|
|
|
|
test("every page of the servers is read, with the token as a Bearer, and only ever with GET", async () => {
|
|
seen.length = 0;
|
|
const r = await adapter.createHetznerAdapter({ token: "tok", apiUrl: url }).read();
|
|
assert.deepEqual(r.servers.map((s) => s.name), ["s1", "s2"]);
|
|
assert.deepEqual([r.volumes?.length, r.images?.map((i) => i.type).sort(), r.prices?.currency], [1, ["backup", "snapshot"], "EUR"]);
|
|
assert.ok(seen.every((s) => s.method === "GET" && s.auth === "Bearer tok"));
|
|
assert.ok(seen.some((s) => s.path.startsWith("/v1/servers?per_page=50&page=2")));
|
|
});
|
|
|
|
test("a list that cannot be asked is unknown (null), not empty; the servers are what must work", async () => {
|
|
volumesStatus = 500;
|
|
const r = await adapter.createHetznerAdapter({ token: "tok", apiUrl: url }).read();
|
|
assert.deepEqual([r.volumes, r.servers.length], [null, 2]);
|
|
assert.equal(hz.estimateCost(r)?.volumes, 0);
|
|
volumesStatus = 200;
|
|
});
|
|
|
|
test("a token that is refused, and a project over its hourly limit, are said", async () => {
|
|
tokenOk = false;
|
|
const ping = await adapter.createHetznerAdapter({ token: "bad", apiUrl: url }).ping();
|
|
assert.equal(ping.ok, false);
|
|
assert.match(ping.error ?? "", /did not accept the token/);
|
|
tokenOk = true;
|
|
rateLimited = true;
|
|
await assert.rejects(() => adapter.createHetznerAdapter({ token: "t", apiUrl: url }).read(), /too many requests/);
|
|
rateLimited = false;
|
|
assert.equal((await adapter.createHetznerAdapter({ token: "t", apiUrl: url }).ping()).ok, true);
|
|
assert.equal((await adapter.createHetznerAdapter({ token: "t", apiUrl: "http://127.0.0.1:1/v1" }).ping()).ok, false);
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("what is carried between reads", () => {
|
|
const t0 = new Date("2026-06-01T10:00:00Z");
|
|
test("a server first seen starts its clocks now; one seen running remembers when", () => {
|
|
const a = hz.carryTimes([server({ status: "running" }), server({ id: 2, name: "off", status: "off" })], [], t0);
|
|
assert.deepEqual([a[0].lastRunningAt, a[0].statusSince, a[1].lastRunningAt, a[1].statusSince], [t0.toISOString(), t0.toISOString(), null, t0.toISOString()]);
|
|
});
|
|
|
|
test("when it goes off the last time it ran is kept, and the status clock starts at the first read that showed it", () => {
|
|
const t1 = new Date(t0.getTime() + 5 * MIN);
|
|
const t2 = new Date(t0.getTime() + 10 * MIN);
|
|
const running = hz.carryTimes([server()], [], t0);
|
|
const off1 = hz.carryTimes([server({ status: "off" })], running, t1);
|
|
const off2 = hz.carryTimes([server({ status: "off" })], off1, t2);
|
|
assert.deepEqual([off1[0].lastRunningAt, off1[0].statusSince], [t0.toISOString(), t1.toISOString()]);
|
|
assert.deepEqual([off2[0].lastRunningAt, off2[0].statusSince], [t0.toISOString(), t1.toISOString()], "the clock does not restart while the status is the same");
|
|
});
|
|
|
|
test("rescue mode's clock starts when it is first seen on, and stops when it is off", () => {
|
|
const t1 = new Date(t0.getTime() + 5 * MIN);
|
|
const on = hz.carryTimes([server({ rescueEnabled: true })], hz.carryTimes([server()], [], t0), t0);
|
|
const still = hz.carryTimes([server({ rescueEnabled: true })], on, t1);
|
|
assert.equal(still[0].rescueSince, t0.toISOString());
|
|
assert.equal(hz.carryTimes([server()], still, t1)[0].rescueSince, null);
|
|
});
|
|
});
|
|
|
|
describe("the cost estimate", () => {
|
|
test("servers, their backups (a percentage of the server), volumes, snapshots and primary IPs, at list prices", () => {
|
|
const c = hz.estimateCost({
|
|
servers: [server({ monthlyPrice: 10, backupWindow: "22-02" }), server({ id: 2, name: "b", monthlyPrice: 5 })],
|
|
volumes: [{ id: 1, name: "v", size: 100, serverId: 1, location: "fsn1", status: "available" }],
|
|
images: [{ id: 1, type: "snapshot", description: null, created: null, sizeGb: 10, serverId: 1 }, { id: 2, type: "backup", description: null, created: null, sizeGb: 30, serverId: 1 }],
|
|
primaryIps: [{ id: 1, ip: "1.1.1.1", type: "ipv4", location: "fsn1", assigned: true }, { id: 2, ip: "1.1.1.2", type: "ipv4", location: "fsn1", assigned: false }],
|
|
prices: PRICES,
|
|
})!;
|
|
assert.deepEqual({ ...c }, { currency: "EUR", servers: 15, backups: 2, volumes: 5, snapshots: 0.12, ips: 1, total: 23.12, unassignedIps: 1, unpriced: 0 });
|
|
});
|
|
|
|
test("without the price list there is no estimate; a server whose price is not known is counted as unpriced", () => {
|
|
assert.equal(hz.estimateCost(read({ prices: null })), null);
|
|
assert.equal(hz.estimateCost(read({ servers: [server({ monthlyPrice: null })] }))?.unpriced, 1);
|
|
});
|
|
});
|
|
|
|
describe("what is wrong", () => {
|
|
const NOW = Date.parse("2026-06-10T12:00:00Z");
|
|
const stored = (extra: Partial<Server_> = {}, times: Partial<import("../src/services/hetznerSync.js").StoredServer> = {}) => ({ ...server({ created: iso(NOW - 30 * 24 * HOUR), ...extra }), lastRunningAt: iso(NOW - HOUR), statusSince: iso(NOW - HOUR), rescueSince: null, ...times });
|
|
const keys = (list: ReturnType<typeof hz.hetznerProblems>) => list.map((p) => `${p.key}=${p.severity}`);
|
|
const problems = (s: ReturnType<typeof stored>, images: Read["images"] = [], hours = 48) => hz.hetznerProblems(1, "Prod", [s], images, NOW, hours);
|
|
|
|
test("a server that is running and well says nothing", () => {
|
|
assert.deepEqual(problems(stored()), []);
|
|
});
|
|
|
|
test("a server that was running and is off is a warning; one off for a week, or never seen running, is left alone", () => {
|
|
assert.deepEqual(keys(problems(stored({ status: "off" }))), ["hetzner-off:1:1=warning"]);
|
|
assert.match(problems(stored({ status: "off" }))[0].message, /Prod: server web is off — it was running 1 hour ago \(Hetzner bills a server that is off\)/);
|
|
assert.deepEqual(problems(stored({ status: "off" }, { lastRunningAt: iso(NOW - 8 * 24 * HOUR) })), []);
|
|
assert.deepEqual(problems(stored({ status: "off" }, { lastRunningAt: null })), []);
|
|
});
|
|
|
|
test("a status that should pass is a problem after half an hour, and not before", () => {
|
|
assert.deepEqual(problems(stored({ status: "migrating" }, { statusSince: iso(NOW - 10 * MIN) })), []);
|
|
const p = problems(stored({ status: "migrating" }, { statusSince: iso(NOW - 45 * MIN) }));
|
|
assert.deepEqual(keys(p), ["hetzner-stuck:1:1=warning"]);
|
|
assert.match(p[0].message, /has been "migrating" for 45 minutes/);
|
|
});
|
|
|
|
test("rescue mode left on for an hour is a warning, and says what a reboot would do", () => {
|
|
assert.deepEqual(problems(stored({ rescueEnabled: true }, { rescueSince: iso(NOW - 30 * MIN) })), []);
|
|
const p = problems(stored({ rescueEnabled: true }, { rescueSince: iso(NOW - 3 * HOUR) }));
|
|
assert.deepEqual(keys(p), ["hetzner-rescue:1:1=warning"]);
|
|
assert.match(p[0].message, /next reboot would come up in the rescue system/);
|
|
});
|
|
|
|
test("traffic: a warning from 80% of the month's quota, critical at 100%, in the units the page shows", () => {
|
|
assert.deepEqual(problems(stored({ outgoingTraffic: 15 * TB })), []);
|
|
const warn = problems(stored({ outgoingTraffic: 17 * TB }));
|
|
assert.deepEqual(keys(warn), ["hetzner-traffic:1:1=warning"]);
|
|
assert.match(warn[0].message, /web has sent 17\.0 TB of its 20\.0 TB traffic this month \(85%\)/);
|
|
assert.deepEqual(keys(problems(stored({ outgoingTraffic: 21 * TB }))), ["hetzner-traffic:1:1=critical"]);
|
|
assert.deepEqual(problems(stored({ outgoingTraffic: null })), []);
|
|
assert.deepEqual(problems(stored({ includedTraffic: null, outgoingTraffic: 99 * TB })), []);
|
|
});
|
|
|
|
describe("backups", () => {
|
|
const backup = (ageHours: number, serverId = 1) => ({ id: 1, type: "backup" as const, description: null, created: iso(NOW - ageHours * HOUR), sizeGb: 2, serverId });
|
|
test("enabled and recent is fine; enabled and old, or none for an old server, is a warning; not enabled is not judged", () => {
|
|
const on = stored({ backupWindow: "22-02" });
|
|
assert.deepEqual(problems(on, [backup(20)]), []);
|
|
assert.match(problems(on, [backup(100)])[0].message, /backups are enabled for web but the newest is 4 days old/);
|
|
assert.match(problems(on, [])[0].message, /but none has been made/);
|
|
assert.deepEqual(problems(on, [backup(100, 99)]).length, 1, "another server's backup does not count");
|
|
assert.deepEqual(problems(stored(), []), []);
|
|
});
|
|
test("a new server has had no time to make one; the limit is the setting, and 0 turns it off; an unknown list says nothing", () => {
|
|
const fresh = stored({ backupWindow: "22-02", created: iso(NOW - 5 * HOUR) });
|
|
assert.deepEqual(problems(fresh, []), []);
|
|
assert.deepEqual(problems(stored({ backupWindow: "22-02" }), [backup(100)], 0), []);
|
|
assert.deepEqual(problems(stored({ backupWindow: "22-02" }), [backup(100)], 200), []);
|
|
assert.deepEqual(problems(stored({ backupWindow: "22-02" }), null), []);
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("which of the app's servers a Hetzner server is", () => {
|
|
const app = [{ id: 1, name: "web-prod", ips: ["203.0.113.5", "192.168.1.5"] }, { id: 2, name: "Db", ips: [] }, { id: 3, name: "other", ips: ["10.0.0.2"] }];
|
|
test("by an address first, then by name; a server the app does not know matches none", () => {
|
|
assert.equal(hz.matchAppServer({ name: "anything", ipv4: "203.0.113.5", privateIps: [] }, app)?.id, 1);
|
|
assert.equal(hz.matchAppServer({ name: "x", ipv4: null, privateIps: ["10.0.0.2"] }, app)?.id, 3, "a private address counts");
|
|
assert.equal(hz.matchAppServer({ name: "db", ipv4: "198.51.100.9", privateIps: [] }, app)?.id, 2, "names are compared without regard to case");
|
|
assert.equal(hz.matchAppServer({ name: "web", ipv4: "198.51.100.9", privateIps: [] }, app), null);
|
|
});
|
|
test("an address wins over a name that belongs to another server", () => {
|
|
assert.equal(hz.matchAppServer({ name: "Db", ipv4: "203.0.113.5", privateIps: [] }, app)?.id, 1);
|
|
});
|
|
});
|
|
|
|
describe("what is stored", () => {
|
|
let id = 0;
|
|
before(async () => {
|
|
id = (await db.insert(integrations).values({ type: "hetzner", name: "Project", baseUrl: "https://api.hetzner.cloud", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
|
});
|
|
const row = async () => (await db.select().from(hetznerState).where(eq(hetznerState.integrationId, id)))[0];
|
|
after(async () => {
|
|
await hz.forgetHetzner(id);
|
|
await db.delete(integrations).where(eq(integrations.id, id)); // so its servers are not matched by the tests that follow
|
|
});
|
|
|
|
test("the state is replaced at each read, carrying what was seen between them; an unknown list is stored as null", async () => {
|
|
const t0 = new Date();
|
|
await hz.storeRead(id, read({ volumes: null, prices: null }), t0);
|
|
let r = await row();
|
|
assert.deepEqual([hz.serversOf(r).length, hz.volumesOf(r), hz.costOf(r)], [1, null, null]);
|
|
await hz.storeRead(id, read({ servers: [server({ status: "off" })], volumes: [{ id: 1, name: "v", size: 10, serverId: 1, location: "fsn1", status: "available" }] }), new Date(t0.getTime() + 5 * MIN));
|
|
r = await row();
|
|
const [s] = hz.serversOf(r);
|
|
assert.deepEqual([s.status, s.lastRunningAt, hz.volumesOf(r)?.length, hz.costOf(r)?.currency], ["off", t0.toISOString(), 1, "EUR"]);
|
|
});
|
|
|
|
test("a read that failed is said, and what was stored is left alone", async () => {
|
|
const ok = await hz.syncHetzner(id, new Date(), async () => read({ servers: [server({ name: "kept" })] }));
|
|
assert.equal(ok.ok, true);
|
|
const failed = await hz.syncHetzner(id, new Date(), async () => {
|
|
throw new Error("connection refused");
|
|
});
|
|
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
|
|
assert.equal(failed.at, ok.at);
|
|
assert.equal(hz.serversOf(await row())[0].name, "kept");
|
|
assert.equal(hz.isHetznerFresh((await hz.hetznerSyncStatuses())[id], Date.now()), false);
|
|
});
|
|
});
|
|
|
|
describe("the health pass, the page and /metrics", () => {
|
|
let hzId = 0;
|
|
let app: Server;
|
|
let base = "";
|
|
const ids: Record<string, number> = {};
|
|
const fresh = () => setInternalFlag("hetznerSync", JSON.stringify({ [hzId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
|
const stale = () => setInternalFlag("hetznerSync", JSON.stringify({ [hzId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
|
|
|
|
before(async () => {
|
|
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
|
hzId = (await db.insert(integrations).values({ type: "hetzner", name: "Prod project", baseUrl: "https://api.hetzner.cloud", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
|
const mk = async (name: string, ips: string[]) => (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, ipAddresses: JSON.stringify(ips) }).returning({ id: servers.id }))[0].id;
|
|
ids.web = await mk("web-agent", ["203.0.113.5"]);
|
|
ids.other = await mk("elsewhere", ["192.168.9.9"]);
|
|
// web: running but nearly out of traffic; db: was running and is now off; blog: rescue left on for hours.
|
|
const t0 = new Date(Date.now() - 3 * HOUR);
|
|
await hz.storeRead(hzId, read({ servers: [server({ id: 1, name: "web", outgoingTraffic: 19 * TB, backupWindow: "22-02" }), server({ id: 2, name: "db", ipv4: "203.0.113.6", status: "running" }), server({ id: 3, name: "blog", ipv4: "203.0.113.7", rescueEnabled: true })], volumes: [{ id: 1, name: "data", size: 100, serverId: 1, location: "fsn1", status: "available" }, { id: 2, name: "orphan", size: 20, serverId: null, location: "fsn1", status: "available" }], images: [{ id: 1, type: "snapshot", description: "before upgrade", created: iso(Date.now() - 40 * 24 * HOUR), sizeGb: 8, serverId: 1 }] }), t0);
|
|
await hz.storeRead(hzId, read({ servers: [server({ id: 1, name: "web", outgoingTraffic: 19 * TB, backupWindow: "22-02" }), server({ id: 2, name: "db", ipv4: "203.0.113.6", status: "off" }), server({ id: 3, name: "blog", ipv4: "203.0.113.7", rescueEnabled: true })], volumes: [{ id: 1, name: "data", size: 100, serverId: 1, location: "fsn1", status: "available" }, { id: 2, name: "orphan", size: 20, serverId: null, location: "fsn1", status: "available" }], images: [{ id: 1, type: "snapshot", description: "before upgrade", created: iso(Date.now() - 40 * 24 * HOUR), sizeGb: 8, serverId: 1 }] }), new Date());
|
|
await fresh();
|
|
const a = express();
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
|
next();
|
|
});
|
|
a.use("/api/hetzner", hetznerRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/hetzner`;
|
|
});
|
|
after(() => {
|
|
app.close();
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows keeps the database file locked until the process exits.
|
|
}
|
|
});
|
|
|
|
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
|
|
let { snapshot, held } = await collectSnapshot();
|
|
assert.deepEqual(snapshot.hetzner?.map((h) => h.integrationId), [hzId]);
|
|
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
|
const hetzner = list.filter((x) => x.key.startsWith("hetzner-")).map((x) => `${x.key}=${x.severity}`).sort();
|
|
assert.deepEqual(hetzner, [`hetzner-backup:${hzId}:1=warning`, `hetzner-off:${hzId}:2=warning`, `hetzner-rescue:${hzId}:3=warning`, `hetzner-traffic:${hzId}:1=warning`].sort());
|
|
const off = list.find((x) => x.key === `hetzner-off:${hzId}:2`)!;
|
|
assert.equal(off.source, `hetzner:${hzId}`);
|
|
for (const k of ["off", "stuck", "rescue", "traffic", "backup"]) assert.equal(subjectOfConditionKey(`hetzner-${k}:${hzId}:2`), `integration:${hzId}`, "maintenance on the project silences its alerts");
|
|
await stale();
|
|
({ snapshot, held } = await collectSnapshot());
|
|
assert.deepEqual([snapshot.hetzner ?? [], held.has(`hetzner:${hzId}`)], [[], true]);
|
|
await fresh();
|
|
});
|
|
|
|
test("the Alerts page shows them with where to look: the backup under Backups, the traffic under Monitoring, the rest under Services", async () => {
|
|
invalidateAlertsCache();
|
|
const report = await getAlerts(true);
|
|
const find = (kind: string, server: number) => report.alerts.find((x) => x.id.endsWith(`hetzner-${kind}:${hzId}:${server}`))!;
|
|
assert.deepEqual([find("off", 2).category, find("off", 2).source, find("off", 2).link, find("off", 2).target?.type], ["services", "Hetzner", "/hetzner", "integration"]);
|
|
assert.equal(find("traffic", 1).category, "monitoring");
|
|
assert.equal(find("backup", 1).category, "backup");
|
|
assert.equal(find("rescue", 3).category, "services");
|
|
});
|
|
|
|
test("the page: each server with the app's server it is (or none), the volumes with what they hang on, the snapshots, the cost and what is wrong", async () => {
|
|
const body = (await (await fetch(base)).json()) as { projects: { id: number; fresh: boolean; problems: unknown[]; state: { servers: { name: string; linked: { id: number; name: string } | null }[]; volumes: { name: string; serverName: string | null }[] | null; images: { description: string; serverName: string | null }[]; cost: { currency: string; total: number } } }[] };
|
|
const p = body.projects.find((x) => x.id === hzId)!;
|
|
assert.equal(p.fresh, true);
|
|
assert.deepEqual(p.state.servers.map((s) => [s.name, s.linked?.name ?? null]), [["web", "web-agent"], ["db", null], ["blog", null]]);
|
|
assert.deepEqual(p.state.volumes?.map((v) => [v.name, v.serverName]), [["data", "web"], ["orphan", null]]);
|
|
assert.deepEqual(p.state.images.map((i) => [i.description, i.serverName]), [["before upgrade", "web"]]);
|
|
assert.equal(p.state.cost.currency, "EUR");
|
|
assert.equal(p.problems.length, 4);
|
|
await stale();
|
|
const old = (await (await fetch(base)).json()) as typeof body;
|
|
assert.deepEqual(old.projects.find((x) => x.id === hzId)!.problems, [], "an old reading says nothing is wrong");
|
|
await fresh();
|
|
});
|
|
|
|
test("the server's own page gets its Hetzner server, with what is wrong with it, and a server that is not one gets none", async () => {
|
|
const mine = (await (await fetch(`${base}/for-server/${ids.web}`)).json()) as { match: { integrationName: string; fresh: boolean; server: { name: string; includedTraffic: number }; problems: { key: string }[]; snapshots: number; newestBackup: string | null } | null };
|
|
assert.deepEqual([mine.match?.integrationName, mine.match?.server.name, mine.match?.snapshots, mine.match?.newestBackup], ["Prod project", "web", 1, null]);
|
|
assert.deepEqual(mine.match?.problems.map((p) => p.key).sort(), [`hetzner-backup:${hzId}:1`, `hetzner-traffic:${hzId}:1`]);
|
|
const none = (await (await fetch(`${base}/for-server/${ids.other}`)).json()) as { match: unknown };
|
|
assert.equal(none.match, null);
|
|
assert.equal((await fetch(`${base}/for-server/999999`)).status, 404);
|
|
assert.equal((await fetch(`${base}/for-server/abc`)).status, 400);
|
|
});
|
|
|
|
test("/metrics has whether each server runs, its traffic against the quota and the project's cost — only while the reading is recent", async () => {
|
|
let text = await buildMetrics(new Date());
|
|
assert.match(text, /homelab_hetzner_server_running\{project="Prod project",server="web"\} 1/);
|
|
assert.match(text, /homelab_hetzner_server_running\{project="Prod project",server="db"\} 0/);
|
|
assert.match(text, /homelab_hetzner_traffic_included_bytes\{project="Prod project",server="web"\}/);
|
|
assert.match(text, /homelab_hetzner_monthly_cost\{[^}]*currency="EUR"[^}]*\} \d/);
|
|
await stale();
|
|
text = await buildMetrics(new Date());
|
|
assert.doesNotMatch(text, /homelab_hetzner/);
|
|
await fresh();
|
|
});
|
|
|
|
test("taking the integration away takes what was read with it", async () => {
|
|
await hz.forgetHetzner(hzId);
|
|
assert.equal((await hz.hetznerSyncStatuses())[hzId], undefined);
|
|
await db.delete(integrations).where(eq(integrations.id, hzId));
|
|
assert.deepEqual(await db.select().from(hetznerState).where(eq(hetznerState.integrationId, hzId)), []);
|
|
});
|
|
});
|