/** * Hetzner Robot: reading what its webservice says (dedicated servers, storage boxes, whether a rescue system is on), what is stored, what is wrong (about to shut down, locked, a rescue system left on, * stuck being set up), the health pass, the alerts, the page's data and /metrics. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { createServer as createHttpServer, type Server } from "node:http"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-robot-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, integrations, hetznerRobotState } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const adapter = await import("../src/integrations/hetznerRobot/adapter.js"); const robot = await import("../src/services/hetznerRobotSync.js"); const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js"); const { subjectOfConditionKey } = await import("../src/services/maintenance.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { hetznerRobotRouter } = await import("../src/routes/hetznerRobot.js"); const { buildMetrics } = await import("../src/services/metrics.js"); const { setInternalFlag } = await import("../src/services/settingsStore.js"); const { INTEGRATION_FIELDS, validateIntegrationConfig, resolveBaseUrl } = await import("../src/integrations/fieldSchemas.js"); const { createIntegrationAdapter } = await import("../src/integrations/registry.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); const MIN = 60_000; const HOUR = 60 * MIN; const DAY = 24 * HOUR; const iso = (ms: number) => new Date(ms).toISOString(); /** A date "YYYY-MM-DD" this many days from `base`. */ const dateIn = (days: number, base = Date.now()) => iso(base + days * DAY).slice(0, 10); type Server_ = import("../src/integrations/hetznerRobot/adapter.js").RobotServer; type Box = import("../src/integrations/hetznerRobot/adapter.js").RobotStorageBox; type Read = import("../src/integrations/hetznerRobot/adapter.js").RobotRead; const server = (extra: Partial = {}): Server_ => ({ number: 100, name: "metal-1", ip: "203.0.113.5", ipv6: "2001:db8::", product: "AX41-NVMe", dc: "FSN1-DC14", traffic: "unlimited", status: "ready", cancelled: false, paidUntil: null, rescue: false, ...extra }); const box = (extra: Partial = {}): Box => ({ id: 7, name: "backups", login: "u123456", product: "BX20", location: "FSN1", cancelled: false, locked: false, paidUntil: null, linkedServer: null, ...extra }); const read = (extra: Partial = {}): Read => ({ servers: [server()], storageBoxes: [box()], skippedDetail: 0, ...extra }); // What Robot sends: each entry wrapped in an object of its own kind. const apiServer = (n: number, extra: Record = {}) => ({ server: { server_ip: `203.0.113.${n}`, server_ipv6_net: "2001:db8::", server_number: n, server_name: `metal-${n}`, product: "AX41-NVMe", dc: "FSN1-DC14", traffic: "unlimited", status: "ready", cancelled: false, paid_until: "2027-01-01", ...extra } }); describe("what Robot says", () => { test("a server: number, name, product, data centre, addresses, traffic plan, status, cancelled and until when — wrapped or not", () => { const s = adapter.parseServer(apiServer(5, { cancelled: true, paid_until: "2026-12-01", status: "In Process" }))!; assert.deepEqual([s.number, s.name, s.ip, s.ipv6, s.product, s.dc, s.traffic, s.status, s.cancelled, s.paidUntil, s.rescue], [5, "metal-5", "203.0.113.5", "2001:db8::", "AX41-NVMe", "FSN1-DC14", "unlimited", "in process", true, "2026-12-01", null]); assert.equal(adapter.parseServer(apiServer(5).server)!.number, 5, "a bare object too"); assert.equal(adapter.parseServer({ server: { server_number: 9, rescue: true } })!.rescue, true); assert.equal(adapter.parseServer({ server: { server_number: 9, server_name: "", product: "EX44" } })!.name, "EX44"); assert.equal(adapter.parseServer({ server: { server_number: 9 } })!.name, "Server 9"); assert.equal(adapter.parseServer({ server: { server_name: "no number" } }), null); assert.equal(adapter.parseServer({ server: { server_number: 1, paid_until: "soon" } })!.paidUntil, null); assert.equal(adapter.parseServer({ server: { server_number: 1, paid_until: "2026-12-01 10:00:00" } })!.paidUntil, "2026-12-01"); }); test("a storage box: id, name, login, product, location, cancelled, locked, paid until, the server it is linked to", () => { const b = adapter.parseStorageBox({ storagebox: { id: 3, login: "u1", name: "Photos", product: "BX40", location: "HEL1", cancelled: true, locked: true, paid_until: "2026-11-30", linked_server: 100 } })!; assert.deepEqual([b.id, b.name, b.login, b.product, b.location, b.cancelled, b.locked, b.paidUntil, b.linkedServer], [3, "Photos", "u1", "BX40", "HEL1", true, true, "2026-11-30", 100]); assert.equal(adapter.parseStorageBox({ storagebox: { id: 4, login: "u2" } })!.name, "u2"); assert.equal(adapter.parseStorageBox({ storagebox: { login: "no id" } }), null); }); describe("the adapter against a Robot that answers", () => { let fake: Server; let url = ""; let servers = [apiServer(2), apiServer(1)]; let listStatus = 200; let boxStatus = 200; let detailFails = new Set(); const seen: { method: string | undefined; path: string; auth: string | undefined }[] = []; before(async () => { fake = createHttpServer((req, res) => { const u = new URL(req.url ?? "", "http://x"); seen.push({ method: req.method, path: u.pathname, auth: req.headers.authorization }); const send = (code: number, body: unknown) => { res.writeHead(code, { "content-type": "application/json" }); res.end(JSON.stringify(body)); }; const fail = (code: number, c: string) => send(code, { error: { status: code, code: c, message: c } }); if (req.headers.authorization !== `Basic ${Buffer.from("#ws+abc:p@ss:word").toString("base64")}`) return fail(401, "UNAUTHORIZED"); if (u.pathname === "/server") return listStatus === 200 ? (servers.length === 0 ? fail(404, "SERVER_NOT_FOUND") : send(200, servers)) : fail(listStatus, "RATE_LIMIT_EXCEEDED"); const one = /^\/server\/(\d+)$/.exec(u.pathname); if (one) return detailFails.has(Number(one[1])) ? fail(500, "INTERNAL_ERROR") : send(200, apiServer(Number(one[1]), { rescue: one[1] === "2" })); if (u.pathname === "/storagebox") return boxStatus === 200 ? send(200, [{ storagebox: { id: 7, login: "u1", name: "backups", cancelled: false, locked: false } }]) : boxStatus === 404 ? fail(404, "STORAGEBOX_NOT_FOUND") : fail(boxStatus, "INTERNAL_ERROR"); fail(404, "NOT_FOUND"); }); await new Promise((resolve) => fake.listen(0, "127.0.0.1", resolve)); url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}`; }); after(() => fake.close()); const make = (extra: Record = {}) => adapter.createHetznerRobotAdapter({ user: "#ws+abc", password: "p@ss:word", apiUrl: url, ...extra }); test("the servers by number, each asked about for its rescue system, and the storage boxes — with the webservice user as Basic auth, and only ever GET", async () => { seen.length = 0; const r = await make().read(); assert.deepEqual(r.servers.map((s) => [s.number, s.rescue]), [[1, false], [2, true]]); assert.deepEqual([r.storageBoxes?.map((b) => b.name), r.skippedDetail], [["backups"], 0]); assert.ok(seen.every((s) => s.method === "GET")); assert.deepEqual(seen.map((s) => s.path).sort(), ["/server", "/server/1", "/server/2", "/storagebox"]); }); test("an account with nothing in it is answered with a 404 and a code ending in NOT_FOUND, which is none — not a failure", async () => { servers = []; boxStatus = 404; const r = await make().read(); assert.deepEqual([r.servers, r.storageBoxes], [[], []]); assert.equal((await make().ping()).ok, true); servers = [apiServer(2), apiServer(1)]; boxStatus = 200; }); test("a part that cannot be read is null (the storage boxes) or unknown (a server's rescue system), not empty or off; the list of servers is what must work", async () => { boxStatus = 500; detailFails = new Set([2]); const r = await make().read(); assert.deepEqual([r.storageBoxes, r.servers.map((s) => s.rescue)], [null, [false, null]]); boxStatus = 200; detailFails = new Set(); listStatus = 403; await assert.rejects(() => make().read(), /refused the request \(HTTP 403, RATE_LIMIT_EXCEEDED\).*200 requests/); listStatus = 200; }); test("only the first 25 servers are asked about, to stay under Robot's 200 requests an hour; the rest are counted", async () => { servers = Array.from({ length: 27 }, (_, i) => apiServer(i + 1)); seen.length = 0; const r = await make().read(); assert.deepEqual([r.servers.length, r.skippedDetail, seen.filter((s) => /^\/server\/\d+$/.test(s.path)).length], [27, 2, 25]); assert.deepEqual([r.servers[24].rescue, r.servers[25].rescue], [false, null]); servers = [apiServer(2), apiServer(1)]; }); test("a wrong password and an address that is not Robot are said", async () => { const bad = await make({ password: "nope" }).ping(); assert.equal(bad.ok, false); assert.match(bad.error ?? "", /did not accept the user name and password \(HTTP 401\).*webservice user/); assert.equal((await make().ping()).ok, true); assert.equal((await adapter.createHetznerRobotAdapter({ user: "a", password: "b", apiUrl: "http://127.0.0.1:1" }).ping()).ok, false); }); }); }); describe("what is wrong", () => { const NOW = Date.parse("2026-06-10T12:00:00Z"); const stored = (extra: Partial = {}): import("../src/services/hetznerRobotSync.js").StoredRobot => ({ servers: robot.carrySince([server()], null, new Date(NOW)), storageBoxes: [box()], skippedDetail: 0, updatedAt: iso(NOW), ...extra, }); const withServer = (extra: Partial) => stored({ servers: [{ ...robot.carrySince([server()], null, new Date(NOW))[0], ...extra }] }); const keys = (list: ReturnType) => list.map((p) => `${p.key}=${p.severity}`); test("an account that is well says nothing", () => { assert.deepEqual(robot.robotProblems(1, "Hetzner", stored(), NOW), []); assert.deepEqual(robot.robotProblems(1, "Hetzner", stored({ storageBoxes: null }), NOW), []); }); test("days left are calendar days to the date: 0 on the day itself, negative once it has passed", () => { assert.equal(robot.daysLeft("2026-06-10", NOW), 0, "ends today"); assert.equal(robot.daysLeft("2026-06-09", NOW), -1); assert.equal(robot.daysLeft("2026-06-10", Date.parse("2026-06-10T23:30:00Z")), 0, "late in the day it is still today"); assert.equal(robot.daysLeft("2026-07-10", NOW), 30); assert.deepEqual([robot.daysLeft(null, NOW), robot.daysLeft("nonsense", NOW)], [null, null]); }); describe("a cancelled server", () => { const cancelled = (days: number) => withServer({ cancelled: true, paidUntil: dateIn(days, NOW) }); test("within 30 days is a warning, within 7 critical; further off, or not cancelled, is nothing", () => { assert.deepEqual(robot.robotProblems(1, "H", cancelled(45), NOW), []); assert.deepEqual(keys(robot.robotProblems(1, "H", cancelled(30), NOW)), ["hetznerrobot-ending:1:100=warning"]); assert.deepEqual(keys(robot.robotProblems(1, "H", cancelled(10), NOW)), ["hetznerrobot-ending:1:100=warning"]); assert.deepEqual(keys(robot.robotProblems(1, "H", cancelled(6), NOW)), ["hetznerrobot-ending:1:100=critical"]); assert.deepEqual(robot.robotProblems(1, "H", withServer({ cancelled: false, paidUntil: dateIn(2, NOW) }), NOW), [], "paid until soon is only a renewal"); assert.deepEqual(robot.robotProblems(1, "H", withServer({ cancelled: true, paidUntil: null }), NOW), [], "no date, no days"); }); test("the message names the server, says when and warns that the data goes", () => { const m = robot.robotProblems(1, "Hetzner", cancelled(10), NOW)[0].message; assert.match(m, /^Hetzner: metal-1 \(server 100\) is cancelled and ends in 10 days \(\d{4}-\d{2}-\d{2}\) — the server and its data go with it$/); assert.match(robot.robotProblems(1, "H", cancelled(0), NOW)[0].message, /is cancelled and ends today \(/); assert.match(robot.robotProblems(1, "H", cancelled(1), NOW)[0].message, /is cancelled and ends tomorrow \(/); assert.match(robot.robotProblems(1, "H", withServer({ cancelled: true, paidUntil: "2026-06-01" }), NOW)[0].message, /has ended/); }); }); describe("a rescue system left on, and a server stuck being set up", () => { test("rescue on for two hours is raised; for less is not; off or unknown never is", () => { const on = (ago: number) => withServer({ rescue: true, rescueSince: iso(NOW - ago) }); const p = robot.robotProblems(1, "H", on(5 * HOUR), NOW); assert.deepEqual(keys(p), ["hetznerrobot-rescue:1:100=warning"]); assert.equal(p[0].message, "H: the rescue system of metal-1 (server 100) has been switched on for 5 hours — its next boot is into the rescue system, not its own"); assert.deepEqual(robot.robotProblems(1, "H", on(30 * MIN), NOW), []); assert.deepEqual(robot.robotProblems(1, "H", withServer({ rescue: null, rescueSince: iso(NOW - DAY) }), NOW), []); assert.deepEqual(robot.robotProblems(1, "H", withServer({ rescue: false, rescueSince: null }), NOW), []); }); test("not ready for a day is raised, and says what it is", () => { const p = robot.robotProblems(1, "H", withServer({ status: "in process", notReadySince: iso(NOW - 3 * DAY) }), NOW); assert.deepEqual(keys(p), ["hetznerrobot-setup:1:100=warning"]); assert.equal(p[0].message, "H: metal-1 (server 100) has been “in process” for 3 days instead of ready"); assert.deepEqual(robot.robotProblems(1, "H", withServer({ status: "in process", notReadySince: iso(NOW - 2 * HOUR) }), NOW), []); }); test("what is seen between reads: the clock starts at the first read that shows it, is kept while it goes on, and is dropped when it is over", () => { const t0 = new Date(NOW); const t1 = new Date(NOW + 20 * MIN); const first = robot.carrySince([server({ number: 1, rescue: true }), server({ number: 2, status: "in process" })], null, t0); assert.deepEqual(first.map((s) => [s.rescueSince, s.notReadySince]), [[t0.toISOString(), null], [null, t0.toISOString()]]); const second = robot.carrySince([server({ number: 1, rescue: true }), server({ number: 2, status: "ready" }), server({ number: 3, rescue: true })], first, t1); assert.deepEqual(second.map((s) => [s.rescueSince, s.notReadySince]), [[t0.toISOString(), null], [null, null], [t1.toISOString(), null]]); }); }); describe("storage boxes", () => { test("a locked one is critical; one cancelled within 30 days is a warning, within 7 critical", () => { assert.deepEqual(keys(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ locked: true })] }), NOW)), ["hetznerrobot-boxlocked:1:7=critical"]); assert.deepEqual(keys(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ cancelled: true, paidUntil: dateIn(20, NOW) })] }), NOW)), ["hetznerrobot-boxending:1:7=warning"]); assert.deepEqual(keys(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ cancelled: true, paidUntil: dateIn(3, NOW) })] }), NOW)), ["hetznerrobot-boxending:1:7=critical"]); assert.deepEqual(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ cancelled: true, paidUntil: dateIn(90, NOW) })] }), NOW), []); assert.match(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ locked: true })] }), NOW)[0].message, /^H: storage box backups \(u123456\) is locked — its data cannot be reached$/); }); }); }); describe("what is stored", () => { let id = 0; before(async () => { id = (await db.insert(integrations).values({ type: "hetznerrobot", name: "Metal", baseUrl: "https://robot-ws.your-server.de", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id; }); const row = async () => (await db.select().from(hetznerRobotState).where(eq(hetznerRobotState.integrationId, id)))[0]; after(async () => { await robot.forgetRobot(id); await db.delete(integrations).where(eq(integrations.id, id)); }); test("the state is replaced at each read; storage boxes that could not be read are stored as null", async () => { await robot.storeRead(id, read({ storageBoxes: null, skippedDetail: 3 }), new Date()); let s = robot.storedOf(await row()); assert.deepEqual([s.storageBoxes, s.servers.length, s.skippedDetail], [null, 1, 3]); await robot.storeRead(id, read({ servers: [server(), server({ number: 101, name: "metal-2" })] }), new Date()); s = robot.storedOf(await row()); assert.deepEqual([s.servers.map((x) => x.name), s.storageBoxes?.length], [["metal-1", "metal-2"], 1]); }); test("since when the rescue system has been on is carried from read to read, and dropped when it is off", async () => { const t0 = new Date(Date.now() - 5 * HOUR); await robot.storeRead(id, read({ servers: [server({ rescue: true })] }), t0); await robot.storeRead(id, read({ servers: [server({ rescue: true })] }), new Date()); assert.equal(robot.storedOf(await row()).servers[0].rescueSince, t0.toISOString()); await robot.storeRead(id, read(), new Date()); assert.equal(robot.storedOf(await row()).servers[0].rescueSince, null); }); test("a read that failed is said, and what was stored is left alone", async () => { const ok = await robot.syncRobot(id, new Date(), async () => read({ servers: [server({ name: "kept" })] })); assert.equal(ok.ok, true); const failed = await robot.syncRobot(id, new Date(), async () => { throw new Error("connection refused"); }); assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]); assert.equal(failed.at, ok.at); assert.equal(robot.storedOf(await row()).servers[0].name, "kept"); assert.equal(robot.isRobotFresh((await robot.robotSyncStatuses())[id], Date.now()), false); }); test("an integration of another type is not read as Robot", async () => { const other = (await db.insert(integrations).values({ type: "gitea", name: "G", baseUrl: "http://g", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id; const status = await robot.syncRobot(other, new Date(), async () => read()); assert.deepEqual([status.ok, status.error], [false, "not a Hetzner Robot integration"]); await robot.forgetRobot(other); await db.delete(integrations).where(eq(integrations.id, other)); }); }); describe("adding one", () => { test("it needs a webservice user and its password (kept secret), and no address — Robot has one", () => { assert.deepEqual(validateIntegrationConfig("hetznerrobot", { user: "#ws+x", password: "p" }), []); assert.deepEqual(validateIntegrationConfig("hetznerrobot", {}), ["user", "password"]); assert.equal(INTEGRATION_FIELDS.hetznerrobot!.find((f) => f.key === "password")!.secret, true); assert.equal(resolveBaseUrl("hetznerrobot", {}), "https://robot-ws.your-server.de"); const a = createIntegrationAdapter("hetznerrobot", { user: "u", password: "p" } as never) as unknown as Record; assert.deepEqual([typeof a.ping, typeof a.read], ["function", "function"]); }); }); describe("the health pass, the page and /metrics", () => { let aId = 0; let app: Server; let base = ""; const fresh = () => setInternalFlag("hetznerRobotSync", JSON.stringify({ [aId]: { at: new Date().toISOString(), ok: true, error: null } })); const stale = () => setInternalFlag("hetznerRobotSync", JSON.stringify({ [aId]: { at: new Date(Date.now() - 60 * MIN).toISOString(), ok: true, error: null } })); before(async () => { await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]); aId = (await db.insert(integrations).values({ type: "hetznerrobot", name: "Metal", baseUrl: "https://robot-ws.your-server.de", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id; const now = Date.now(); // One server cancelled and ending in four days, one with the rescue system on for five hours, one fine; a locked storage box and one cancelled further off. const servers = [server({ number: 100, name: "metal-1", cancelled: true, paidUntil: dateIn(4, now) }), server({ number: 101, name: "metal-2", rescue: true }), server({ number: 102, name: "metal-3" })]; const boxes = [box({ id: 7, locked: true }), box({ id: 8, name: "old", cancelled: true, paidUntil: dateIn(90, now) })]; await robot.storeRead(aId, read({ servers, storageBoxes: boxes }), new Date(now - 5 * HOUR)); await robot.storeRead(aId, read({ servers, storageBoxes: boxes }), new Date(now)); await fresh(); const a = express(); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: "viewer" } }; next(); }); a.use("/api/hetzner-robot", hetznerRobotRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/hetzner-robot`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("the health pass judges the account from what is stored, and holds it when the last read is old", async () => { let { snapshot, held } = await collectSnapshot(); assert.deepEqual(snapshot.hetznerRobot?.map((r) => r.integrationId), [aId]); const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true }); const mine = list.filter((x) => x.key.startsWith("hetznerrobot-")); assert.deepEqual(mine.map((x) => x.key).sort(), [`hetznerrobot-ending:${aId}:100`, `hetznerrobot-rescue:${aId}:101`, `hetznerrobot-boxlocked:${aId}:7`].sort()); const ending = mine.find((x) => x.key === `hetznerrobot-ending:${aId}:100`)!; assert.deepEqual([ending.severity, ending.source], ["critical", `hetznerrobot:${aId}`]); for (const k of ["ending", "rescue", "setup", "boxlocked", "boxending"]) assert.equal(subjectOfConditionKey(`hetznerrobot-${k}:${aId}:1`), `integration:${aId}`); await stale(); ({ snapshot, held } = await collectSnapshot()); assert.deepEqual([snapshot.hetznerRobot, held.has(`hetznerrobot:${aId}`)], [undefined, true]); await fresh(); }); test("the Alerts page puts them under Services, names the source, and links to the Hetzner Robot page", async () => { invalidateAlertsCache(); const report = await getAlerts(true); const a = report.alerts.find((x) => x.id.endsWith(`hetznerrobot-ending:${aId}:100`))!; const locked = report.alerts.find((x) => x.id.endsWith(`hetznerrobot-boxlocked:${aId}:7`))!; assert.deepEqual([a.category, a.source, a.link, a.target?.type, locked.category, locked.severity], ["services", "Hetzner Robot", "/hetzner-robot", "integration", "services", "critical"]); }); test("the page: the account with its servers (rescue, cancellation with days left), storage boxes and what is wrong", async () => { type Body = { accounts: { id: number; fresh: boolean; problems: { key: string }[]; state: { servers: { number: number; rescue: boolean | null; rescueSince: string | null }[]; storageBoxes: { id: number; locked: boolean }[] | null; serverDays: Record; boxDays: Record } }[] }; const body = (await (await fetch(base)).json()) as Body; const a = body.accounts.find((x) => x.id === aId)!; assert.deepEqual([a.fresh, a.state.servers.map((s) => s.number), a.problems.length, Object.keys(a.state.serverDays), a.state.storageBoxes?.map((b) => [b.id, b.locked])], [true, [100, 101, 102], 3, ["100"], [[7, true], [8, false]]]); assert.ok(a.state.serverDays["100"]! >= 3 && a.state.serverDays["100"]! <= 5); assert.deepEqual(Object.keys(a.state.boxDays), ["8"]); assert.ok(a.state.servers.find((s) => s.number === 101)!.rescueSince); await stale(); const old = (await (await fetch(base)).json()) as Body; assert.deepEqual(old.accounts.find((x) => x.id === aId)!.problems, [], "an old reading says nothing is wrong"); await fresh(); }); test("/metrics has the servers, the rescue systems, the days left and the locked boxes — only while the reading is recent", async () => { let text = await buildMetrics(new Date()); assert.match(text, /homelab_hetzner_robot_servers\{name="Metal"\} 3/); assert.match(text, /homelab_hetzner_robot_rescue_active\{name="Metal",server="101",server_name="metal-2"\} 1/); assert.match(text, /homelab_hetzner_robot_rescue_active\{name="Metal",server="100",server_name="metal-1"\} 0/); assert.match(text, /homelab_hetzner_robot_days_left\{name="Metal",server="100",server_name="metal-1"\} [345]/); assert.doesNotMatch(text, /homelab_hetzner_robot_days_left\{name="Metal",server="102"/, "not cancelled, no days"); assert.match(text, /homelab_hetzner_robot_storage_box_locked\{name="Metal",storage_box="7",storage_box_name="backups"\} 1/); assert.match(text, /homelab_hetzner_robot_days_left\{name="Metal",storage_box="8",storage_box_name="old"\} (89|90|91)/); await stale(); text = await buildMetrics(new Date()); assert.doesNotMatch(text, /homelab_hetzner_robot_/); await fresh(); }); test("taking the integration away takes what was read with it", async () => { await robot.forgetRobot(aId); assert.equal((await robot.robotSyncStatuses())[aId], undefined); await db.delete(integrations).where(eq(integrations.id, aId)); assert.deepEqual(await db.select().from(hetznerRobotState).where(eq(hetznerRobotState.integrationId, aId)), []); }); });