Add Hetzner Robot: dedicated servers and storage boxes, with shutting-down, locked and rescue-left-on alerts
Read every 15 minutes (Robot allows 200 requests an hour; read-only): each server's product, addresses, status, rescue system and paid-until, and the storage boxes. Adapter, sync service, hetzner_robot_state table, /api/hetzner-robot, Infrastructure → Hetzner Robot page, metrics and docs. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7618bf0f0a
commit
01acbaf831
31 files changed
+10105
-10
No files matched your search
@@ -0,0 +1,394 @@
|
||||
/**
|
||||
* Hetzner Robot: reading what its webservice says (dedicated servers, storage boxes, whether a rescue system is on), what is stored, what is wrong (about to shut down, locked, a rescue system left on,
|
||||
* stuck being set up), the health pass, the alerts, the page's data and /metrics.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-robot-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, integrations, hetznerRobotState } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const adapter = await import("../src/integrations/hetznerRobot/adapter.js");
|
||||
const robot = await import("../src/services/hetznerRobotSync.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { hetznerRobotRouter } = await import("../src/routes/hetznerRobot.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { INTEGRATION_FIELDS, validateIntegrationConfig, resolveBaseUrl } = await import("../src/integrations/fieldSchemas.js");
|
||||
const { createIntegrationAdapter } = await import("../src/integrations/registry.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
const DAY = 24 * HOUR;
|
||||
const iso = (ms: number) => new Date(ms).toISOString();
|
||||
/** A date "YYYY-MM-DD" this many days from `base`. */
|
||||
const dateIn = (days: number, base = Date.now()) => iso(base + days * DAY).slice(0, 10);
|
||||
|
||||
type Server_ = import("../src/integrations/hetznerRobot/adapter.js").RobotServer;
|
||||
type Box = import("../src/integrations/hetznerRobot/adapter.js").RobotStorageBox;
|
||||
type Read = import("../src/integrations/hetznerRobot/adapter.js").RobotRead;
|
||||
|
||||
const server = (extra: Partial<Server_> = {}): Server_ => ({ number: 100, name: "metal-1", ip: "203.0.113.5", ipv6: "2001:db8::", product: "AX41-NVMe", dc: "FSN1-DC14", traffic: "unlimited", status: "ready", cancelled: false, paidUntil: null, rescue: false, ...extra });
|
||||
const box = (extra: Partial<Box> = {}): Box => ({ id: 7, name: "backups", login: "u123456", product: "BX20", location: "FSN1", cancelled: false, locked: false, paidUntil: null, linkedServer: null, ...extra });
|
||||
const read = (extra: Partial<Read> = {}): Read => ({ servers: [server()], storageBoxes: [box()], skippedDetail: 0, ...extra });
|
||||
|
||||
// What Robot sends: each entry wrapped in an object of its own kind.
|
||||
const apiServer = (n: number, extra: Record<string, unknown> = {}) => ({ server: { server_ip: `203.0.113.${n}`, server_ipv6_net: "2001:db8::", server_number: n, server_name: `metal-${n}`, product: "AX41-NVMe", dc: "FSN1-DC14", traffic: "unlimited", status: "ready", cancelled: false, paid_until: "2027-01-01", ...extra } });
|
||||
|
||||
describe("what Robot says", () => {
|
||||
test("a server: number, name, product, data centre, addresses, traffic plan, status, cancelled and until when — wrapped or not", () => {
|
||||
const s = adapter.parseServer(apiServer(5, { cancelled: true, paid_until: "2026-12-01", status: "In Process" }))!;
|
||||
assert.deepEqual([s.number, s.name, s.ip, s.ipv6, s.product, s.dc, s.traffic, s.status, s.cancelled, s.paidUntil, s.rescue], [5, "metal-5", "203.0.113.5", "2001:db8::", "AX41-NVMe", "FSN1-DC14", "unlimited", "in process", true, "2026-12-01", null]);
|
||||
assert.equal(adapter.parseServer(apiServer(5).server)!.number, 5, "a bare object too");
|
||||
assert.equal(adapter.parseServer({ server: { server_number: 9, rescue: true } })!.rescue, true);
|
||||
assert.equal(adapter.parseServer({ server: { server_number: 9, server_name: "", product: "EX44" } })!.name, "EX44");
|
||||
assert.equal(adapter.parseServer({ server: { server_number: 9 } })!.name, "Server 9");
|
||||
assert.equal(adapter.parseServer({ server: { server_name: "no number" } }), null);
|
||||
assert.equal(adapter.parseServer({ server: { server_number: 1, paid_until: "soon" } })!.paidUntil, null);
|
||||
assert.equal(adapter.parseServer({ server: { server_number: 1, paid_until: "2026-12-01 10:00:00" } })!.paidUntil, "2026-12-01");
|
||||
});
|
||||
|
||||
test("a storage box: id, name, login, product, location, cancelled, locked, paid until, the server it is linked to", () => {
|
||||
const b = adapter.parseStorageBox({ storagebox: { id: 3, login: "u1", name: "Photos", product: "BX40", location: "HEL1", cancelled: true, locked: true, paid_until: "2026-11-30", linked_server: 100 } })!;
|
||||
assert.deepEqual([b.id, b.name, b.login, b.product, b.location, b.cancelled, b.locked, b.paidUntil, b.linkedServer], [3, "Photos", "u1", "BX40", "HEL1", true, true, "2026-11-30", 100]);
|
||||
assert.equal(adapter.parseStorageBox({ storagebox: { id: 4, login: "u2" } })!.name, "u2");
|
||||
assert.equal(adapter.parseStorageBox({ storagebox: { login: "no id" } }), null);
|
||||
});
|
||||
|
||||
describe("the adapter against a Robot that answers", () => {
|
||||
let fake: Server;
|
||||
let url = "";
|
||||
let servers = [apiServer(2), apiServer(1)];
|
||||
let listStatus = 200;
|
||||
let boxStatus = 200;
|
||||
let detailFails = new Set<number>();
|
||||
const seen: { method: string | undefined; path: string; auth: string | undefined }[] = [];
|
||||
before(async () => {
|
||||
fake = createHttpServer((req, res) => {
|
||||
const u = new URL(req.url ?? "", "http://x");
|
||||
seen.push({ method: req.method, path: u.pathname, auth: req.headers.authorization });
|
||||
const send = (code: number, body: unknown) => {
|
||||
res.writeHead(code, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify(body));
|
||||
};
|
||||
const fail = (code: number, c: string) => send(code, { error: { status: code, code: c, message: c } });
|
||||
if (req.headers.authorization !== `Basic ${Buffer.from("#ws+abc:p@ss:word").toString("base64")}`) return fail(401, "UNAUTHORIZED");
|
||||
if (u.pathname === "/server") return listStatus === 200 ? (servers.length === 0 ? fail(404, "SERVER_NOT_FOUND") : send(200, servers)) : fail(listStatus, "RATE_LIMIT_EXCEEDED");
|
||||
const one = /^\/server\/(\d+)$/.exec(u.pathname);
|
||||
if (one) return detailFails.has(Number(one[1])) ? fail(500, "INTERNAL_ERROR") : send(200, apiServer(Number(one[1]), { rescue: one[1] === "2" }));
|
||||
if (u.pathname === "/storagebox") return boxStatus === 200 ? send(200, [{ storagebox: { id: 7, login: "u1", name: "backups", cancelled: false, locked: false } }]) : boxStatus === 404 ? fail(404, "STORAGEBOX_NOT_FOUND") : fail(boxStatus, "INTERNAL_ERROR");
|
||||
fail(404, "NOT_FOUND");
|
||||
});
|
||||
await new Promise<void>((resolve) => fake.listen(0, "127.0.0.1", resolve));
|
||||
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => fake.close());
|
||||
const make = (extra: Record<string, unknown> = {}) => adapter.createHetznerRobotAdapter({ user: "#ws+abc", password: "p@ss:word", apiUrl: url, ...extra });
|
||||
|
||||
test("the servers by number, each asked about for its rescue system, and the storage boxes — with the webservice user as Basic auth, and only ever GET", async () => {
|
||||
seen.length = 0;
|
||||
const r = await make().read();
|
||||
assert.deepEqual(r.servers.map((s) => [s.number, s.rescue]), [[1, false], [2, true]]);
|
||||
assert.deepEqual([r.storageBoxes?.map((b) => b.name), r.skippedDetail], [["backups"], 0]);
|
||||
assert.ok(seen.every((s) => s.method === "GET"));
|
||||
assert.deepEqual(seen.map((s) => s.path).sort(), ["/server", "/server/1", "/server/2", "/storagebox"]);
|
||||
});
|
||||
|
||||
test("an account with nothing in it is answered with a 404 and a code ending in NOT_FOUND, which is none — not a failure", async () => {
|
||||
servers = [];
|
||||
boxStatus = 404;
|
||||
const r = await make().read();
|
||||
assert.deepEqual([r.servers, r.storageBoxes], [[], []]);
|
||||
assert.equal((await make().ping()).ok, true);
|
||||
servers = [apiServer(2), apiServer(1)];
|
||||
boxStatus = 200;
|
||||
});
|
||||
|
||||
test("a part that cannot be read is null (the storage boxes) or unknown (a server's rescue system), not empty or off; the list of servers is what must work", async () => {
|
||||
boxStatus = 500;
|
||||
detailFails = new Set([2]);
|
||||
const r = await make().read();
|
||||
assert.deepEqual([r.storageBoxes, r.servers.map((s) => s.rescue)], [null, [false, null]]);
|
||||
boxStatus = 200;
|
||||
detailFails = new Set();
|
||||
listStatus = 403;
|
||||
await assert.rejects(() => make().read(), /refused the request \(HTTP 403, RATE_LIMIT_EXCEEDED\).*200 requests/);
|
||||
listStatus = 200;
|
||||
});
|
||||
|
||||
test("only the first 25 servers are asked about, to stay under Robot's 200 requests an hour; the rest are counted", async () => {
|
||||
servers = Array.from({ length: 27 }, (_, i) => apiServer(i + 1));
|
||||
seen.length = 0;
|
||||
const r = await make().read();
|
||||
assert.deepEqual([r.servers.length, r.skippedDetail, seen.filter((s) => /^\/server\/\d+$/.test(s.path)).length], [27, 2, 25]);
|
||||
assert.deepEqual([r.servers[24].rescue, r.servers[25].rescue], [false, null]);
|
||||
servers = [apiServer(2), apiServer(1)];
|
||||
});
|
||||
|
||||
test("a wrong password and an address that is not Robot are said", async () => {
|
||||
const bad = await make({ password: "nope" }).ping();
|
||||
assert.equal(bad.ok, false);
|
||||
assert.match(bad.error ?? "", /did not accept the user name and password \(HTTP 401\).*webservice user/);
|
||||
assert.equal((await make().ping()).ok, true);
|
||||
assert.equal((await adapter.createHetznerRobotAdapter({ user: "a", password: "b", apiUrl: "http://127.0.0.1:1" }).ping()).ok, false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is wrong", () => {
|
||||
const NOW = Date.parse("2026-06-10T12:00:00Z");
|
||||
const stored = (extra: Partial<import("../src/services/hetznerRobotSync.js").StoredRobot> = {}): import("../src/services/hetznerRobotSync.js").StoredRobot => ({
|
||||
servers: robot.carrySince([server()], null, new Date(NOW)),
|
||||
storageBoxes: [box()],
|
||||
skippedDetail: 0,
|
||||
updatedAt: iso(NOW),
|
||||
...extra,
|
||||
});
|
||||
const withServer = (extra: Partial<import("../src/services/hetznerRobotSync.js").StoredRobotServer>) => stored({ servers: [{ ...robot.carrySince([server()], null, new Date(NOW))[0], ...extra }] });
|
||||
const keys = (list: ReturnType<typeof robot.robotProblems>) => list.map((p) => `${p.key}=${p.severity}`);
|
||||
|
||||
test("an account that is well says nothing", () => {
|
||||
assert.deepEqual(robot.robotProblems(1, "Hetzner", stored(), NOW), []);
|
||||
assert.deepEqual(robot.robotProblems(1, "Hetzner", stored({ storageBoxes: null }), NOW), []);
|
||||
});
|
||||
|
||||
test("days left are calendar days to the date: 0 on the day itself, negative once it has passed", () => {
|
||||
assert.equal(robot.daysLeft("2026-06-10", NOW), 0, "ends today");
|
||||
assert.equal(robot.daysLeft("2026-06-09", NOW), -1);
|
||||
assert.equal(robot.daysLeft("2026-06-10", Date.parse("2026-06-10T23:30:00Z")), 0, "late in the day it is still today");
|
||||
assert.equal(robot.daysLeft("2026-07-10", NOW), 30);
|
||||
assert.deepEqual([robot.daysLeft(null, NOW), robot.daysLeft("nonsense", NOW)], [null, null]);
|
||||
});
|
||||
|
||||
describe("a cancelled server", () => {
|
||||
const cancelled = (days: number) => withServer({ cancelled: true, paidUntil: dateIn(days, NOW) });
|
||||
test("within 30 days is a warning, within 7 critical; further off, or not cancelled, is nothing", () => {
|
||||
assert.deepEqual(robot.robotProblems(1, "H", cancelled(45), NOW), []);
|
||||
assert.deepEqual(keys(robot.robotProblems(1, "H", cancelled(30), NOW)), ["hetznerrobot-ending:1:100=warning"]);
|
||||
assert.deepEqual(keys(robot.robotProblems(1, "H", cancelled(10), NOW)), ["hetznerrobot-ending:1:100=warning"]);
|
||||
assert.deepEqual(keys(robot.robotProblems(1, "H", cancelled(6), NOW)), ["hetznerrobot-ending:1:100=critical"]);
|
||||
assert.deepEqual(robot.robotProblems(1, "H", withServer({ cancelled: false, paidUntil: dateIn(2, NOW) }), NOW), [], "paid until soon is only a renewal");
|
||||
assert.deepEqual(robot.robotProblems(1, "H", withServer({ cancelled: true, paidUntil: null }), NOW), [], "no date, no days");
|
||||
});
|
||||
test("the message names the server, says when and warns that the data goes", () => {
|
||||
const m = robot.robotProblems(1, "Hetzner", cancelled(10), NOW)[0].message;
|
||||
assert.match(m, /^Hetzner: metal-1 \(server 100\) is cancelled and ends in 10 days \(\d{4}-\d{2}-\d{2}\) — the server and its data go with it$/);
|
||||
assert.match(robot.robotProblems(1, "H", cancelled(0), NOW)[0].message, /is cancelled and ends today \(/);
|
||||
assert.match(robot.robotProblems(1, "H", cancelled(1), NOW)[0].message, /is cancelled and ends tomorrow \(/);
|
||||
assert.match(robot.robotProblems(1, "H", withServer({ cancelled: true, paidUntil: "2026-06-01" }), NOW)[0].message, /has ended/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a rescue system left on, and a server stuck being set up", () => {
|
||||
test("rescue on for two hours is raised; for less is not; off or unknown never is", () => {
|
||||
const on = (ago: number) => withServer({ rescue: true, rescueSince: iso(NOW - ago) });
|
||||
const p = robot.robotProblems(1, "H", on(5 * HOUR), NOW);
|
||||
assert.deepEqual(keys(p), ["hetznerrobot-rescue:1:100=warning"]);
|
||||
assert.equal(p[0].message, "H: the rescue system of metal-1 (server 100) has been switched on for 5 hours — its next boot is into the rescue system, not its own");
|
||||
assert.deepEqual(robot.robotProblems(1, "H", on(30 * MIN), NOW), []);
|
||||
assert.deepEqual(robot.robotProblems(1, "H", withServer({ rescue: null, rescueSince: iso(NOW - DAY) }), NOW), []);
|
||||
assert.deepEqual(robot.robotProblems(1, "H", withServer({ rescue: false, rescueSince: null }), NOW), []);
|
||||
});
|
||||
test("not ready for a day is raised, and says what it is", () => {
|
||||
const p = robot.robotProblems(1, "H", withServer({ status: "in process", notReadySince: iso(NOW - 3 * DAY) }), NOW);
|
||||
assert.deepEqual(keys(p), ["hetznerrobot-setup:1:100=warning"]);
|
||||
assert.equal(p[0].message, "H: metal-1 (server 100) has been “in process” for 3 days instead of ready");
|
||||
assert.deepEqual(robot.robotProblems(1, "H", withServer({ status: "in process", notReadySince: iso(NOW - 2 * HOUR) }), NOW), []);
|
||||
});
|
||||
test("what is seen between reads: the clock starts at the first read that shows it, is kept while it goes on, and is dropped when it is over", () => {
|
||||
const t0 = new Date(NOW);
|
||||
const t1 = new Date(NOW + 20 * MIN);
|
||||
const first = robot.carrySince([server({ number: 1, rescue: true }), server({ number: 2, status: "in process" })], null, t0);
|
||||
assert.deepEqual(first.map((s) => [s.rescueSince, s.notReadySince]), [[t0.toISOString(), null], [null, t0.toISOString()]]);
|
||||
const second = robot.carrySince([server({ number: 1, rescue: true }), server({ number: 2, status: "ready" }), server({ number: 3, rescue: true })], first, t1);
|
||||
assert.deepEqual(second.map((s) => [s.rescueSince, s.notReadySince]), [[t0.toISOString(), null], [null, null], [t1.toISOString(), null]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("storage boxes", () => {
|
||||
test("a locked one is critical; one cancelled within 30 days is a warning, within 7 critical", () => {
|
||||
assert.deepEqual(keys(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ locked: true })] }), NOW)), ["hetznerrobot-boxlocked:1:7=critical"]);
|
||||
assert.deepEqual(keys(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ cancelled: true, paidUntil: dateIn(20, NOW) })] }), NOW)), ["hetznerrobot-boxending:1:7=warning"]);
|
||||
assert.deepEqual(keys(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ cancelled: true, paidUntil: dateIn(3, NOW) })] }), NOW)), ["hetznerrobot-boxending:1:7=critical"]);
|
||||
assert.deepEqual(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ cancelled: true, paidUntil: dateIn(90, NOW) })] }), NOW), []);
|
||||
assert.match(robot.robotProblems(1, "H", stored({ storageBoxes: [box({ locked: true })] }), NOW)[0].message, /^H: storage box backups \(u123456\) is locked — its data cannot be reached$/);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is stored", () => {
|
||||
let id = 0;
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "hetznerrobot", name: "Metal", baseUrl: "https://robot-ws.your-server.de", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
});
|
||||
const row = async () => (await db.select().from(hetznerRobotState).where(eq(hetznerRobotState.integrationId, id)))[0];
|
||||
after(async () => {
|
||||
await robot.forgetRobot(id);
|
||||
await db.delete(integrations).where(eq(integrations.id, id));
|
||||
});
|
||||
|
||||
test("the state is replaced at each read; storage boxes that could not be read are stored as null", async () => {
|
||||
await robot.storeRead(id, read({ storageBoxes: null, skippedDetail: 3 }), new Date());
|
||||
let s = robot.storedOf(await row());
|
||||
assert.deepEqual([s.storageBoxes, s.servers.length, s.skippedDetail], [null, 1, 3]);
|
||||
await robot.storeRead(id, read({ servers: [server(), server({ number: 101, name: "metal-2" })] }), new Date());
|
||||
s = robot.storedOf(await row());
|
||||
assert.deepEqual([s.servers.map((x) => x.name), s.storageBoxes?.length], [["metal-1", "metal-2"], 1]);
|
||||
});
|
||||
|
||||
test("since when the rescue system has been on is carried from read to read, and dropped when it is off", async () => {
|
||||
const t0 = new Date(Date.now() - 5 * HOUR);
|
||||
await robot.storeRead(id, read({ servers: [server({ rescue: true })] }), t0);
|
||||
await robot.storeRead(id, read({ servers: [server({ rescue: true })] }), new Date());
|
||||
assert.equal(robot.storedOf(await row()).servers[0].rescueSince, t0.toISOString());
|
||||
await robot.storeRead(id, read(), new Date());
|
||||
assert.equal(robot.storedOf(await row()).servers[0].rescueSince, null);
|
||||
});
|
||||
|
||||
test("a read that failed is said, and what was stored is left alone", async () => {
|
||||
const ok = await robot.syncRobot(id, new Date(), async () => read({ servers: [server({ name: "kept" })] }));
|
||||
assert.equal(ok.ok, true);
|
||||
const failed = await robot.syncRobot(id, new Date(), async () => {
|
||||
throw new Error("connection refused");
|
||||
});
|
||||
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
|
||||
assert.equal(failed.at, ok.at);
|
||||
assert.equal(robot.storedOf(await row()).servers[0].name, "kept");
|
||||
assert.equal(robot.isRobotFresh((await robot.robotSyncStatuses())[id], Date.now()), false);
|
||||
});
|
||||
|
||||
test("an integration of another type is not read as Robot", async () => {
|
||||
const other = (await db.insert(integrations).values({ type: "gitea", name: "G", baseUrl: "http://g", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
const status = await robot.syncRobot(other, new Date(), async () => read());
|
||||
assert.deepEqual([status.ok, status.error], [false, "not a Hetzner Robot integration"]);
|
||||
await robot.forgetRobot(other);
|
||||
await db.delete(integrations).where(eq(integrations.id, other));
|
||||
});
|
||||
});
|
||||
|
||||
describe("adding one", () => {
|
||||
test("it needs a webservice user and its password (kept secret), and no address — Robot has one", () => {
|
||||
assert.deepEqual(validateIntegrationConfig("hetznerrobot", { user: "#ws+x", password: "p" }), []);
|
||||
assert.deepEqual(validateIntegrationConfig("hetznerrobot", {}), ["user", "password"]);
|
||||
assert.equal(INTEGRATION_FIELDS.hetznerrobot!.find((f) => f.key === "password")!.secret, true);
|
||||
assert.equal(resolveBaseUrl("hetznerrobot", {}), "https://robot-ws.your-server.de");
|
||||
const a = createIntegrationAdapter("hetznerrobot", { user: "u", password: "p" } as never) as unknown as Record<string, unknown>;
|
||||
assert.deepEqual([typeof a.ping, typeof a.read], ["function", "function"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the health pass, the page and /metrics", () => {
|
||||
let aId = 0;
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const fresh = () => setInternalFlag("hetznerRobotSync", JSON.stringify({ [aId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
const stale = () => setInternalFlag("hetznerRobotSync", JSON.stringify({ [aId]: { at: new Date(Date.now() - 60 * MIN).toISOString(), ok: true, error: null } }));
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
aId = (await db.insert(integrations).values({ type: "hetznerrobot", name: "Metal", baseUrl: "https://robot-ws.your-server.de", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
const now = Date.now();
|
||||
// One server cancelled and ending in four days, one with the rescue system on for five hours, one fine; a locked storage box and one cancelled further off.
|
||||
const servers = [server({ number: 100, name: "metal-1", cancelled: true, paidUntil: dateIn(4, now) }), server({ number: 101, name: "metal-2", rescue: true }), server({ number: 102, name: "metal-3" })];
|
||||
const boxes = [box({ id: 7, locked: true }), box({ id: 8, name: "old", cancelled: true, paidUntil: dateIn(90, now) })];
|
||||
await robot.storeRead(aId, read({ servers, storageBoxes: boxes }), new Date(now - 5 * HOUR));
|
||||
await robot.storeRead(aId, read({ servers, storageBoxes: boxes }), new Date(now));
|
||||
await fresh();
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/hetzner-robot", hetznerRobotRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/hetzner-robot`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("the health pass judges the account from what is stored, and holds it when the last read is old", async () => {
|
||||
let { snapshot, held } = await collectSnapshot();
|
||||
assert.deepEqual(snapshot.hetznerRobot?.map((r) => r.integrationId), [aId]);
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const mine = list.filter((x) => x.key.startsWith("hetznerrobot-"));
|
||||
assert.deepEqual(mine.map((x) => x.key).sort(), [`hetznerrobot-ending:${aId}:100`, `hetznerrobot-rescue:${aId}:101`, `hetznerrobot-boxlocked:${aId}:7`].sort());
|
||||
const ending = mine.find((x) => x.key === `hetznerrobot-ending:${aId}:100`)!;
|
||||
assert.deepEqual([ending.severity, ending.source], ["critical", `hetznerrobot:${aId}`]);
|
||||
for (const k of ["ending", "rescue", "setup", "boxlocked", "boxending"]) assert.equal(subjectOfConditionKey(`hetznerrobot-${k}:${aId}:1`), `integration:${aId}`);
|
||||
await stale();
|
||||
({ snapshot, held } = await collectSnapshot());
|
||||
assert.deepEqual([snapshot.hetznerRobot, held.has(`hetznerrobot:${aId}`)], [undefined, true]);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("the Alerts page puts them under Services, names the source, and links to the Hetzner Robot page", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const a = report.alerts.find((x) => x.id.endsWith(`hetznerrobot-ending:${aId}:100`))!;
|
||||
const locked = report.alerts.find((x) => x.id.endsWith(`hetznerrobot-boxlocked:${aId}:7`))!;
|
||||
assert.deepEqual([a.category, a.source, a.link, a.target?.type, locked.category, locked.severity], ["services", "Hetzner Robot", "/hetzner-robot", "integration", "services", "critical"]);
|
||||
});
|
||||
|
||||
test("the page: the account with its servers (rescue, cancellation with days left), storage boxes and what is wrong", async () => {
|
||||
type Body = { accounts: { id: number; fresh: boolean; problems: { key: string }[]; state: { servers: { number: number; rescue: boolean | null; rescueSince: string | null }[]; storageBoxes: { id: number; locked: boolean }[] | null; serverDays: Record<string, number | null>; boxDays: Record<string, number | null> } }[] };
|
||||
const body = (await (await fetch(base)).json()) as Body;
|
||||
const a = body.accounts.find((x) => x.id === aId)!;
|
||||
assert.deepEqual([a.fresh, a.state.servers.map((s) => s.number), a.problems.length, Object.keys(a.state.serverDays), a.state.storageBoxes?.map((b) => [b.id, b.locked])], [true, [100, 101, 102], 3, ["100"], [[7, true], [8, false]]]);
|
||||
assert.ok(a.state.serverDays["100"]! >= 3 && a.state.serverDays["100"]! <= 5);
|
||||
assert.deepEqual(Object.keys(a.state.boxDays), ["8"]);
|
||||
assert.ok(a.state.servers.find((s) => s.number === 101)!.rescueSince);
|
||||
await stale();
|
||||
const old = (await (await fetch(base)).json()) as Body;
|
||||
assert.deepEqual(old.accounts.find((x) => x.id === aId)!.problems, [], "an old reading says nothing is wrong");
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("/metrics has the servers, the rescue systems, the days left and the locked boxes — only while the reading is recent", async () => {
|
||||
let text = await buildMetrics(new Date());
|
||||
assert.match(text, /homelab_hetzner_robot_servers\{name="Metal"\} 3/);
|
||||
assert.match(text, /homelab_hetzner_robot_rescue_active\{name="Metal",server="101",server_name="metal-2"\} 1/);
|
||||
assert.match(text, /homelab_hetzner_robot_rescue_active\{name="Metal",server="100",server_name="metal-1"\} 0/);
|
||||
assert.match(text, /homelab_hetzner_robot_days_left\{name="Metal",server="100",server_name="metal-1"\} [345]/);
|
||||
assert.doesNotMatch(text, /homelab_hetzner_robot_days_left\{name="Metal",server="102"/, "not cancelled, no days");
|
||||
assert.match(text, /homelab_hetzner_robot_storage_box_locked\{name="Metal",storage_box="7",storage_box_name="backups"\} 1/);
|
||||
assert.match(text, /homelab_hetzner_robot_days_left\{name="Metal",storage_box="8",storage_box_name="old"\} (89|90|91)/);
|
||||
await stale();
|
||||
text = await buildMetrics(new Date());
|
||||
assert.doesNotMatch(text, /homelab_hetzner_robot_/);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("taking the integration away takes what was read with it", async () => {
|
||||
await robot.forgetRobot(aId);
|
||||
assert.equal((await robot.robotSyncStatuses())[aId], undefined);
|
||||
await db.delete(integrations).where(eq(integrations.id, aId));
|
||||
assert.deepEqual(await db.select().from(hetznerRobotState).where(eq(hetznerRobotState.integrationId, aId)), []);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user