Add Hetzner Robot: dedicated servers and storage boxes, with shutting-down, locked and rescue-left-on alerts

Read every 15 minutes (Robot allows 200 requests an hour; read-only): each server's product, addresses, status, rescue system and paid-until, and the storage boxes. Adapter, sync service, hetzner_robot_state table, /api/hetzner-robot, Infrastructure → Hetzner Robot page, metrics and docs.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-11-04 23:26:26 +01:00
1 parent 7618bf0f0a
commit 01acbaf831
31 files changed
+10105 -10

No files matched your search

+394
View File
@@ -0,0 +1,394 @@
/**
* Hetzner Robot: reading what its webservice says (dedicated servers, storage boxes, whether a rescue system is on), what is stored, what is wrong (about to shut down, locked, a rescue system left on,
* stuck being set up), the health pass, the alerts, the page's data and /metrics.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-robot-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, hetznerRobotState } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/hetznerRobot/adapter.js");
const robot = await import("../src/services/hetznerRobotSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { hetznerRobotRouter } = await import("../src/routes/hetznerRobot.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { INTEGRATION_FIELDS, validateIntegrationConfig, resolveBaseUrl } = await import("../src/integrations/fieldSchemas.js");
const { createIntegrationAdapter } = await import("../src/integrations/registry.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const HOUR = 60 * MIN;
const DAY = 24 * HOUR;
const iso = (ms: number) => new Date(ms).toISOString();
/** A date "YYYY-MM-DD" this many days from `base`. */
const dateIn = (days: number, base = Date.now()) => iso(base + days * DAY).slice(0, 10);
type Server_ = import("../src/integrations/hetznerRobot/adapter.js").RobotServer;
type Box = import("../src/integrations/hetznerRobot/adapter.js").RobotStorageBox;
type Read = import("../src/integrations/hetznerRobot/adapter.js").RobotRead;
const server = (extra: Partial<Server_> = {}): Server_ => ({ number: 100, name: "metal-1", ip: "203.0.113.5", ipv6: "2001:db8::", product: "AX41-NVMe", dc: "FSN1-DC14", traffic: "unlimited", status: "ready", cancelled: false, paidUntil: null, rescue: false, ...extra });
const box = (extra: Partial<Box> = {}): Box => ({ id: 7, name: "backups", login: "u123456", product: "BX20", location: "FSN1", cancelled: false, locked: false, paidUntil: null, linkedServer: null, ...extra });
const read = (extra: Partial<Read> = {}): Read => ({ servers: [server()], storageBoxes: [box()], skippedDetail: 0, ...extra });
// What Robot sends: each entry wrapped in an object of its own kind.
const apiServer = (n: number, extra: Record<string, unknown> = {}) => ({ server: { server_ip: `203.0.113.${n}`, server_ipv6_net: "2001:db8::", server_number: n, server_name: `metal-${n}`, product: "AX41-NVMe", dc: "FSN1-DC14", traffic: "unlimited", status: "ready", cancelled: false, paid_until: "2027-01-01", ...extra } });
describe("what Robot says", () => {
test("a server: number, name, product, data centre, addresses, traffic plan, status, cancelled and until when — wrapped or not", () => {
const s = adapter.parseServer(apiServer(5, { cancelled: true, paid_until: "2026-12-01", status: "In Process" }))!;
assert.deepEqual([s.number, s.name, s.ip, s.ipv6, s.product, s.dc, s.traffic, s.status, s.cancelled, s.paidUntil, s.rescue], [5, "metal-5", "203.0.113.5", "2001:db8::", "AX41-NVMe", "FSN1-DC14", "unlimited", "in process", true, "2026-12-01", null]);
assert.equal(adapter.parseServer(apiServer(5).server)!.number, 5, "a bare object too");
assert.equal(adapter.parseServer({ server: { server_number: 9, rescue: true } })!.rescue, true);
assert.equal(adapter.parseServer({ server: { server_number: 9, server_name: "", product: "EX44" } })!.name, "EX44");
assert.equal(adapter.parseServer({ server: { server_number: 9 } })!.name, "Server 9");
assert.equal(adapter.parseServer({ server: { server_name: "no number" } }), null);
assert.equal(adapter.parseServer({ server: { server_number: 1, paid_until: "soon" } })!.paidUntil, null);
assert.equal(adapter.parseServer({ server: { server_number: 1, paid_until: "2026-12-01 10:00:00" } })!.paidUntil, "2026-12-01");
});
test("a storage box: id, name, login, product, location, cancelled, locked, paid until, the server it is linked to", () => {
const b = adapter.parseStorageBox({ storagebox: { id: 3, login: "u1", name: "Photos", product: "BX40", location: "HEL1", cancelled: true, locked: true, paid_until: "2026-11-30", linked_server: 100 } })!;
assert.deepEqual([b.id, b.name, b.login, b.product, b.location, b.cancelled, b.locked, b.paidUntil, b.linkedServer], [3, "Photos", "u1", "BX40", "HEL1", true, true, "2026-11-30", 100]);
assert.equal(adapter.parseStorageBox({ storagebox: { id: 4, login: "u2" } })!.name, "u2");
assert.equal(adapter.parseStorageBox({ storagebox: { login: "no id" } }), null);
});
describe("the adapter against a Robot that answers", () => {
let fake: Server;
let url = "";
let servers = [apiServer(2), apiServer(1)];
let listStatus = 200;
let boxStatus = 200;
let detailFails = new Set<number>();
const seen: { method: string | undefined; path: string; auth: string | undefined }[] = [];
before(async () => {
fake = createHttpServer((req, res) => {
const u = new URL(req.url ?? "", "http://x");
seen.push({ method: req.method, path: u.pathname, auth: req.headers.authorization });
const send = (code: number, body: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(body));
};
const fail = (code: number, c: string) => send(code, { error: { status: code, code: c, message: c } });
if (req.headers.authorization !== `Basic ${Buffer.from("#ws+abc:p@ss:word").toString("base64")}`) return fail(401, "UNAUTHORIZED");
if (u.pathname === "/server") return listStatus === 200 ? (servers.length === 0 ? fail(404, "SERVER_NOT_FOUND") : send(200, servers)) : fail(listStatus, "RATE_LIMIT_EXCEEDED");
const one = /^\/server\/(\d+)$/.exec(u.pathname);
if (one) return detailFails.has(Number(one[1])) ? fail(500, "INTERNAL_ERROR") : send(200, apiServer(Number(one[1]), { rescue: one[1] === "2" }));
if (u.pathname === "/storagebox") return boxStatus === 200 ? send(200, [{ storagebox: { id: 7, login: "u1", name: "backups", cancelled: false, locked: false } }]) : boxStatus === 404 ? fail(404, "STORAGEBOX_NOT_FOUND") : fail(boxStatus, "INTERNAL_ERROR");
fail(404, "NOT_FOUND");
});
await new Promise<void>((resolve) => fake.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}`;
});
after(() => fake.close());
const make = (extra: Record<string, unknown> = {}) => adapter.createHetznerRobotAdapter({ user: "#ws+abc", password: "p@ss:word", apiUrl: url, ...extra });
test("the servers by number, each asked about for its rescue system, and the storage boxes — with the webservice user as Basic auth, and only ever GET", async () => {
seen.length = 0;
const r = await make().read();
assert.deepEqual(r.servers.map((s) => [s.number, s.rescue]), [[1, false], [2, true]]);
assert.deepEqual([r.storageBoxes?.map((b) => b.name), r.skippedDetail], [["backups"], 0]);
assert.ok(seen.every((s) => s.method === "GET"));
assert.deepEqual(seen.map((s) => s.path).sort(), ["/server", "/server/1", "/server/2", "/storagebox"]);
});
test("an account with nothing in it is answered with a 404 and a code ending in NOT_FOUND, which is none — not a failure", async () => {
servers = [];
boxStatus = 404;
const r = await make().read();
assert.deepEqual([r.servers, r.storageBoxes], [[], []]);
assert.equal((await make().ping()).ok, true);
servers = [apiServer(2), apiServer(1)];
boxStatus = 200;
});
test("a part that cannot be read is null (the storage boxes) or unknown (a server's rescue system), not empty or off; the list of servers is what must work", async () => {
boxStatus = 500;
detailFails = new Set([2]);
const r = await make().read();
assert.deepEqual([r.storageBoxes, r.servers.map((s) => s.rescue)], [null, [false, null]]);
boxStatus = 200;
detailFails = new Set();
listStatus = 403;
await assert.rejects(() => make().read(), /refused the request \(HTTP 403, RATE_LIMIT_EXCEEDED\).*200 requests/);
listStatus = 200;
});
test("only the first 25 servers are asked about, to stay under Robot's 200 requests an hour; the rest are counted", async () => {
servers = Array.from({ length: 27 }, (_, i) => apiServer(i + 1));
seen.length = 0;
const r = await make().read();
assert.deepEqual([r.servers.length, r.skippedDetail, seen.filter((s) => /^\/server\/\d+$/.test(s.path)).length], [27, 2, 25]);
assert.deepEqual([r.servers[24].rescue, r.servers[25].rescue], [false, null]);
servers = [apiServer(2), apiServer(1)];
});
test("a wrong password and an address that is not Robot are said", async () => {
const bad = await make({ password: "nope" }).ping();
assert.equal(bad.ok, false);
assert.match(bad.error ?? "", /did not accept the user name and password \(HTTP 401\).*webservice user/);
assert.equal((await make().ping()).ok, true);
assert.equal((await adapter.createHetznerRobotAdapter({ user: "a", password: "b", apiUrl: "http://127.0.0.1:1" }).ping()).ok, false);
});
});
});
describe("what is wrong", () => {
const NOW = Date.parse("2026-06-10T12:00:00Z");
const stored = (extra: Partial<import("../src/services/hetznerRobotSync.js").StoredRobot> = {}): import("../src/services/hetznerRobotSync.js").StoredRobot => ({
servers: robot.carrySince([server()], null, new Date(NOW)),
storageBoxes: [box()],
skippedDetail: 0,
updatedAt: iso(NOW),
...extra,
});
const withServer = (extra: Partial<import("../src/services/hetznerRobotSync.js").StoredRobotServer>) => stored({ servers: [{ ...robot.carrySince([server()], null, new Date(NOW))[0], ...extra }] });
const keys = (list: ReturnType<typeof robot.robotProblems>) => list.map((p) => `${p.key}=${p.severity}`);
test("an account that is well says nothing", () => {
assert.deepEqual(robot.robotProblems(1, "Hetzner", stored(), NOW), []);
assert.deepEqual(robot.robotProblems(1, "Hetzner", stored({ storageBoxes: null }), NOW), []);
});
test("days left are calendar days to the date: 0 on the day itself, negative once it has passed", () => {
assert.equal(robot.daysLeft("2026-06-10", NOW), 0, "ends today");
assert.equal(robot.daysLeft("2026-06-09", NOW), -1);
assert.equal(robot.daysLeft("2026-06-10", Date.parse("2026-06-10T23:30:00Z")), 0, "late in the day it is still today");
assert.equal(robot.daysLeft("2026-07-10", NOW), 30);
assert.deepEqual([robot.daysLeft(null, NOW), robot.daysLeft("nonsense", NOW)], [null, null]);
});
describe("a cancelled server", () => {
const cancelled = (days: number) => withServer({ cancelled: true, paidUntil: dateIn(days, NOW) });
test("within 30 days is a warning, within 7 critical; further off, or not cancelled, is nothing", () => {
assert.deepEqual(robot.robotProblems(1, "H", cancelled(45), NOW), []);
assert.deepEqual(keys(robot.robotProblems(1, "H", cancelled(30), NOW)), ["hetznerrobot-ending:1:100=warning"]);
assert.deepEqual(keys(robot.robotProblems(1, "H", cancelled(10), NOW)), ["hetznerrobot-ending:1:100=warning"]);
assert.deepEqual(keys(robot.robotProblems(1, "H", cancelled(6), NOW)), ["hetznerrobot-ending:1:100=critical"]);
assert.deepEqual(robot.robotProblems(1, "H", withServer({ cancelled: false, paidUntil: dateIn(2, NOW) }), NOW), [], "paid until soon is only a renewal");
assert.deepEqual(robot.robotProblems(1, "H", withServer({ cancelled: true, paidUntil: null }), NOW), [], "no date, no days");
});
test("the message names the server, says when and warns that the data goes", () => {
const m = robot.robotProblems(1, "Hetzner", cancelled(10), NOW)[0].message;
assert.match(m, /^Hetzner: metal-1 \(server 100\) is cancelled and ends in 10 days \(\d{4}-\d{2}-\d{2}\) — the server and its data go with it$/);
assert.match(robot.robotProblems(1, "H", cancelled(0), NOW)[0].message, /is cancelled and ends today \(/);
assert.match(robot.robotProblems(1, "H", cancelled(1), NOW)[0].message, /is cancelled and ends tomorrow \(/);
assert.match(robot.robotProblems(1, "H", withServer({ cancelled: true, paidUntil: "2026-06-01" }), NOW)[0].message, /has ended/);
});
});
describe("a rescue system left on, and a server stuck being set up", () => {
test("rescue on for two hours is raised; for less is not; off or unknown never is", () => {
const on = (ago: number) => withServer({ rescue: true, rescueSince: iso(NOW - ago) });
const p = robot.robotProblems(1, "H", on(5 * HOUR), NOW);
assert.deepEqual(keys(p), ["hetznerrobot-rescue:1:100=warning"]);
assert.equal(p[0].message, "H: the rescue system of metal-1 (server 100) has been switched on for 5 hours — its next boot is into the rescue system, not its own");
assert.deepEqual(robot.robotProblems(1, "H", on(30 * MIN), NOW), []);
assert.deepEqual(robot.robotProblems(1, "H", withServer({ rescue: null, rescueSince: iso(NOW - DAY) }), NOW), []);
assert.deepEqual(robot.robotProblems(1, "H", withServer({ rescue: false, rescueSince: null }), NOW), []);
});
test("not ready for a day is raised, and says what it is", () => {
const p = robot.robotProblems(1, "H", withServer({ status: "in process", notReadySince: iso(NOW - 3 * DAY) }), NOW);
assert.deepEqual(keys(p), ["hetznerrobot-setup:1:100=warning"]);
assert.equal(p[0].message, "H: metal-1 (server 100) has been “in process” for 3 days instead of ready");
assert.deepEqual(robot.robotProblems(1, "H", withServer({ status: "in process", notReadySince: iso(NOW - 2 * HOUR) }), NOW), []);
});
test("what is seen between reads: the clock starts at the first read that shows it, is kept while it goes on, and is dropped when it is over", () => {
const t0 = new Date(NOW);
const t1 = new Date(NOW + 20 * MIN);
const first = robot.carrySince([server({ number: 1, rescue: true }), server({ number: 2, status: "in process" })], null, t0);
assert.deepEqual(first.map((s) => [s.rescueSince, s.notReadySince]), [[t0.toISOString(), null], [null, t0.toISOString()]]);
const second = robot.carrySince([server({ number: 1, rescue: true }), server({ number: 2, status: "ready" }), server({ number: 3, rescue: true })], first, t1);
assert.deepEqual(second.map((s) => [s.rescueSince, s.notReadySince]), [[t0.toISOString(), null], [null, null], [t1.toISOString(), null]]);
});
});
describe("storage boxes", () => {
test("a locked one is critical; one cancelled within 30 days is a warning, within 7 critical", () => {
assert.deepEqual(keys(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ locked: true })] }), NOW)), ["hetznerrobot-boxlocked:1:7=critical"]);
assert.deepEqual(keys(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ cancelled: true, paidUntil: dateIn(20, NOW) })] }), NOW)), ["hetznerrobot-boxending:1:7=warning"]);
assert.deepEqual(keys(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ cancelled: true, paidUntil: dateIn(3, NOW) })] }), NOW)), ["hetznerrobot-boxending:1:7=critical"]);
assert.deepEqual(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ cancelled: true, paidUntil: dateIn(90, NOW) })] }), NOW), []);
assert.match(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ locked: true })] }), NOW)[0].message, /^H: storage box backups \(u123456\) is locked — its data cannot be reached$/);
});
});
});
describe("what is stored", () => {
let id = 0;
before(async () => {
id = (await db.insert(integrations).values({ type: "hetznerrobot", name: "Metal", baseUrl: "https://robot-ws.your-server.de", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
});
const row = async () => (await db.select().from(hetznerRobotState).where(eq(hetznerRobotState.integrationId, id)))[0];
after(async () => {
await robot.forgetRobot(id);
await db.delete(integrations).where(eq(integrations.id, id));
});
test("the state is replaced at each read; storage boxes that could not be read are stored as null", async () => {
await robot.storeRead(id, read({ storageBoxes: null, skippedDetail: 3 }), new Date());
let s = robot.storedOf(await row());
assert.deepEqual([s.storageBoxes, s.servers.length, s.skippedDetail], [null, 1, 3]);
await robot.storeRead(id, read({ servers: [server(), server({ number: 101, name: "metal-2" })] }), new Date());
s = robot.storedOf(await row());
assert.deepEqual([s.servers.map((x) => x.name), s.storageBoxes?.length], [["metal-1", "metal-2"], 1]);
});
test("since when the rescue system has been on is carried from read to read, and dropped when it is off", async () => {
const t0 = new Date(Date.now() - 5 * HOUR);
await robot.storeRead(id, read({ servers: [server({ rescue: true })] }), t0);
await robot.storeRead(id, read({ servers: [server({ rescue: true })] }), new Date());
assert.equal(robot.storedOf(await row()).servers[0].rescueSince, t0.toISOString());
await robot.storeRead(id, read(), new Date());
assert.equal(robot.storedOf(await row()).servers[0].rescueSince, null);
});
test("a read that failed is said, and what was stored is left alone", async () => {
const ok = await robot.syncRobot(id, new Date(), async () => read({ servers: [server({ name: "kept" })] }));
assert.equal(ok.ok, true);
const failed = await robot.syncRobot(id, new Date(), async () => {
throw new Error("connection refused");
});
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
assert.equal(failed.at, ok.at);
assert.equal(robot.storedOf(await row()).servers[0].name, "kept");
assert.equal(robot.isRobotFresh((await robot.robotSyncStatuses())[id], Date.now()), false);
});
test("an integration of another type is not read as Robot", async () => {
const other = (await db.insert(integrations).values({ type: "gitea", name: "G", baseUrl: "http://g", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
const status = await robot.syncRobot(other, new Date(), async () => read());
assert.deepEqual([status.ok, status.error], [false, "not a Hetzner Robot integration"]);
await robot.forgetRobot(other);
await db.delete(integrations).where(eq(integrations.id, other));
});
});
describe("adding one", () => {
test("it needs a webservice user and its password (kept secret), and no address — Robot has one", () => {
assert.deepEqual(validateIntegrationConfig("hetznerrobot", { user: "#ws+x", password: "p" }), []);
assert.deepEqual(validateIntegrationConfig("hetznerrobot", {}), ["user", "password"]);
assert.equal(INTEGRATION_FIELDS.hetznerrobot!.find((f) => f.key === "password")!.secret, true);
assert.equal(resolveBaseUrl("hetznerrobot", {}), "https://robot-ws.your-server.de");
const a = createIntegrationAdapter("hetznerrobot", { user: "u", password: "p" } as never) as unknown as Record<string, unknown>;
assert.deepEqual([typeof a.ping, typeof a.read], ["function", "function"]);
});
});
describe("the health pass, the page and /metrics", () => {
let aId = 0;
let app: Server;
let base = "";
const fresh = () => setInternalFlag("hetznerRobotSync", JSON.stringify({ [aId]: { at: new Date().toISOString(), ok: true, error: null } }));
const stale = () => setInternalFlag("hetznerRobotSync", JSON.stringify({ [aId]: { at: new Date(Date.now() - 60 * MIN).toISOString(), ok: true, error: null } }));
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
aId = (await db.insert(integrations).values({ type: "hetznerrobot", name: "Metal", baseUrl: "https://robot-ws.your-server.de", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
const now = Date.now();
// One server cancelled and ending in four days, one with the rescue system on for five hours, one fine; a locked storage box and one cancelled further off.
const servers = [server({ number: 100, name: "metal-1", cancelled: true, paidUntil: dateIn(4, now) }), server({ number: 101, name: "metal-2", rescue: true }), server({ number: 102, name: "metal-3" })];
const boxes = [box({ id: 7, locked: true }), box({ id: 8, name: "old", cancelled: true, paidUntil: dateIn(90, now) })];
await robot.storeRead(aId, read({ servers, storageBoxes: boxes }), new Date(now - 5 * HOUR));
await robot.storeRead(aId, read({ servers, storageBoxes: boxes }), new Date(now));
await fresh();
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/hetzner-robot", hetznerRobotRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/hetzner-robot`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass judges the account from what is stored, and holds it when the last read is old", async () => {
let { snapshot, held } = await collectSnapshot();
assert.deepEqual(snapshot.hetznerRobot?.map((r) => r.integrationId), [aId]);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const mine = list.filter((x) => x.key.startsWith("hetznerrobot-"));
assert.deepEqual(mine.map((x) => x.key).sort(), [`hetznerrobot-ending:${aId}:100`, `hetznerrobot-rescue:${aId}:101`, `hetznerrobot-boxlocked:${aId}:7`].sort());
const ending = mine.find((x) => x.key === `hetznerrobot-ending:${aId}:100`)!;
assert.deepEqual([ending.severity, ending.source], ["critical", `hetznerrobot:${aId}`]);
for (const k of ["ending", "rescue", "setup", "boxlocked", "boxending"]) assert.equal(subjectOfConditionKey(`hetznerrobot-${k}:${aId}:1`), `integration:${aId}`);
await stale();
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.hetznerRobot, held.has(`hetznerrobot:${aId}`)], [undefined, true]);
await fresh();
});
test("the Alerts page puts them under Services, names the source, and links to the Hetzner Robot page", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const a = report.alerts.find((x) => x.id.endsWith(`hetznerrobot-ending:${aId}:100`))!;
const locked = report.alerts.find((x) => x.id.endsWith(`hetznerrobot-boxlocked:${aId}:7`))!;
assert.deepEqual([a.category, a.source, a.link, a.target?.type, locked.category, locked.severity], ["services", "Hetzner Robot", "/hetzner-robot", "integration", "services", "critical"]);
});
test("the page: the account with its servers (rescue, cancellation with days left), storage boxes and what is wrong", async () => {
type Body = { accounts: { id: number; fresh: boolean; problems: { key: string }[]; state: { servers: { number: number; rescue: boolean | null; rescueSince: string | null }[]; storageBoxes: { id: number; locked: boolean }[] | null; serverDays: Record<string, number | null>; boxDays: Record<string, number | null> } }[] };
const body = (await (await fetch(base)).json()) as Body;
const a = body.accounts.find((x) => x.id === aId)!;
assert.deepEqual([a.fresh, a.state.servers.map((s) => s.number), a.problems.length, Object.keys(a.state.serverDays), a.state.storageBoxes?.map((b) => [b.id, b.locked])], [true, [100, 101, 102], 3, ["100"], [[7, true], [8, false]]]);
assert.ok(a.state.serverDays["100"]! >= 3 && a.state.serverDays["100"]! <= 5);
assert.deepEqual(Object.keys(a.state.boxDays), ["8"]);
assert.ok(a.state.servers.find((s) => s.number === 101)!.rescueSince);
await stale();
const old = (await (await fetch(base)).json()) as Body;
assert.deepEqual(old.accounts.find((x) => x.id === aId)!.problems, [], "an old reading says nothing is wrong");
await fresh();
});
test("/metrics has the servers, the rescue systems, the days left and the locked boxes — only while the reading is recent", async () => {
let text = await buildMetrics(new Date());
assert.match(text, /homelab_hetzner_robot_servers\{name="Metal"\} 3/);
assert.match(text, /homelab_hetzner_robot_rescue_active\{name="Metal",server="101",server_name="metal-2"\} 1/);
assert.match(text, /homelab_hetzner_robot_rescue_active\{name="Metal",server="100",server_name="metal-1"\} 0/);
assert.match(text, /homelab_hetzner_robot_days_left\{name="Metal",server="100",server_name="metal-1"\} [345]/);
assert.doesNotMatch(text, /homelab_hetzner_robot_days_left\{name="Metal",server="102"/, "not cancelled, no days");
assert.match(text, /homelab_hetzner_robot_storage_box_locked\{name="Metal",storage_box="7",storage_box_name="backups"\} 1/);
assert.match(text, /homelab_hetzner_robot_days_left\{name="Metal",storage_box="8",storage_box_name="old"\} (89|90|91)/);
await stale();
text = await buildMetrics(new Date());
assert.doesNotMatch(text, /homelab_hetzner_robot_/);
await fresh();
});
test("taking the integration away takes what was read with it", async () => {
await robot.forgetRobot(aId);
assert.equal((await robot.robotSyncStatuses())[aId], undefined);
await db.delete(integrations).where(eq(integrations.id, aId));
assert.deepEqual(await db.select().from(hetznerRobotState).where(eq(hetznerRobotState.integrationId, aId)), []);
});
});