Add Hetzner Robot: dedicated servers and storage boxes, with shutting-down, locked and rescue-left-on alerts

Read every 15 minutes (Robot allows 200 requests an hour; read-only): each server's product, addresses, status, rescue system and paid-until, and the storage boxes. Adapter, sync service, hetzner_robot_state table, /api/hetzner-robot, Infrastructure → Hetzner Robot page, metrics and docs.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-11-04 23:26:26 +01:00
1 parent 7618bf0f0a
commit 01acbaf831
31 files changed
+10105 -10

No files matched your search

+2
View File
@@ -74,6 +74,7 @@ import { wallRouter } from "./routes/wall.js";
import { adguardRouter } from "./routes/adguard.js";
import { technitiumRouter } from "./routes/technitium.js";
import { downloadsRouter } from "./routes/downloads.js";
import { hetznerRobotRouter } from "./routes/hetznerRobot.js";
import { grafanaRouter } from "./routes/grafana.js";
import { versionsRouter } from "./routes/versions.js";
import { portPresetsRouter } from "./routes/portPresets.js";
@@ -229,6 +230,7 @@ export function createApp() {
app.use("/api/adguard", adguardRouter);
app.use("/api/technitium", technitiumRouter);
app.use("/api/downloads", downloadsRouter);
app.use("/api/hetzner-robot", hetznerRobotRouter);
app.use("/api/grafana", grafanaRouter);
app.use("/api/versions", versionsRouter);
app.use("/api/port-presets", portPresetsRouter);
+16
View File
@@ -580,6 +580,7 @@ export const integrationTypes = [
"readarr",
"qbittorrent",
"sabnzbd",
"hetznerrobot",
] as const;
export type IntegrationType = (typeof integrationTypes)[number];
@@ -1392,6 +1393,21 @@ export const downloadState = sqliteTable("download_state", {
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── Hetzner Robot ──────────────────────────────────────────────────────────
// How a Hetzner Robot account (an integration of type "hetznerrobot") is right now, one row each, replaced at every read: its dedicated servers (each with since when its rescue system has been on and
// since when it has not been "ready", which only this app can know) and its storage boxes. A column that is null could not be read, which is not the same as none. Read by
// services/hetznerRobotSync.ts.
export const hetznerRobotState = sqliteTable("hetzner_robot_state", {
integrationId: integer("integration_id")
.primaryKey()
.references(() => integrations.id, { onDelete: "cascade" }),
servers: text("servers").notNull().default("[]"), // JSON: the dedicated servers, with rescueSince and notReadySince carried from read to read
storageBoxes: text("storage_boxes"), // JSON; null: could not be read
skippedDetail: integer("skipped_detail").notNull().default(0), // servers not asked about their rescue system (too many for Robot's request limit)
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── AdGuard Home ───────────────────────────────────────────────────────────
// How an AdGuard Home (an integration of type "adguard") is right now, one row each, replaced at every read: whether its DNS server runs and protects, the statistics, its filter lists and the result
+2
View File
@@ -45,6 +45,7 @@ import { initArrSync } from "./services/arrSync.js";
import { initAdguardSync } from "./services/adguardSync.js";
import { initTechnitiumSync } from "./services/technitiumSync.js";
import { initDownloadsSync } from "./services/downloadsSync.js";
import { initHetznerRobotSync } from "./services/hetznerRobotSync.js";
import { initGrafanaSync } from "./services/grafanaSync.js";
import { initVersionCheck } from "./services/appVersions.js";
import { backfillMetricRollups } from "./services/serverMetrics.js";
@@ -113,6 +114,7 @@ initArrSync();
initAdguardSync();
initTechnitiumSync();
initDownloadsSync();
initHetznerRobotSync();
initGrafanaSync();
initVersionCheck();
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
+5
View File
@@ -116,6 +116,10 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
hetzner: [{ key: "token", label: "API token", secret: true, type: "password", placeholder: "Hetzner Console → the project → Security → API tokens; Read is enough" }],
hetznerrobot: [
{ key: "user", label: "Webservice user name", secret: false, placeholder: "#ws+… (Robot → Settings → Webservice and app settings)" },
{ key: "password", label: "Webservice password", secret: true, type: "password", placeholder: "the webservice user's own password, not the Robot login" },
],
sonarr: [
{ key: "url", label: "Sonarr URL", secret: false, placeholder: "http://sonarr.example.lan:8989" },
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "Settings → General → Security → API Key" },
@@ -255,6 +259,7 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
export const INTEGRATION_BASE_URLS: Partial<Record<IntegrationType, string>> = {
tailscale: "https://api.tailscale.com",
hetzner: "https://api.hetzner.cloud",
hetznerrobot: "https://robot-ws.your-server.de",
fragnet: "https://panel.fragnet.net",
};
@@ -0,0 +1,179 @@
/**
* Hetzner Robot adapter — Hetzner's dedicated servers and storage boxes (Robot is the other Hetzner, next to the Cloud). Config: user and password of a **webservice user** (Robot → Settings →
* Webservice and app settings; not the login to Robot itself), sent as HTTP Basic to `https://robot-ws.your-server.de`.
*
* Reading only: the account's dedicated servers (number, name, product, data centre, addresses, traffic plan, status, whether it is cancelled and until when, and — by asking for each one — whether the
* rescue system is switched on) and its storage boxes (product, location, whether cancelled or locked, until when it is paid). Nothing is sent to Robot but GET requests: no server is reset, no rescue
* system is activated, nothing is ordered or cancelled.
*
* Robot allows 200 requests an hour, so a read is a list, one request for each server (at most MAX_DETAIL of them) and a list of storage boxes. Endpoints and field names are from Robot's published
* webservice documentation (`/server`, `/server/{number}`, `/storagebox`; each list entry wrapped in an object of its own kind) and have not been checked against a live account here; every field is read
* defensively. A list with nothing in it is answered with a 404 and a code ending in NOT_FOUND, which is read as "none".
*/
import { requestJson } from "../httpJson.js";
import { withDiagLogging } from "../../services/diagLog.js";
export interface HetznerRobotConfig {
user: string;
password: string;
/** Only for tests: the form never asks for it (a request cannot set it — see fieldSchemas.ts). */
apiUrl?: string;
}
export interface RobotServer {
number: number;
name: string;
ip: string | null;
ipv6: string | null;
product: string | null;
/** The data centre ("FSN1-DC14"). */
dc: string | null;
/** The traffic plan as Robot words it ("unlimited", "20 TB"). */
traffic: string | null;
/** "ready", or "in process" while it is being set up. */
status: string;
cancelled: boolean;
/** The date it is paid until, "YYYY-MM-DD"; for a cancelled server, the day it shuts down. */
paidUntil: string | null;
/** Whether the rescue system is switched on for the next boot; null when it could not be asked. */
rescue: boolean | null;
}
export interface RobotStorageBox {
id: number;
name: string;
login: string | null;
product: string | null;
location: string | null;
cancelled: boolean;
locked: boolean;
paidUntil: string | null;
/** The number of the server it is linked to, when it is. */
linkedServer: number | null;
}
export interface RobotRead {
servers: RobotServer[];
/** Null when the storage boxes could not be read (which is not the same as having none). */
storageBoxes: RobotStorageBox[] | null;
/** How many servers were not asked about their rescue system, for being more than MAX_DETAIL. */
skippedDetail: number;
}
export interface HetznerRobotAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
read(): Promise<RobotRead>;
}
const API = "https://robot-ws.your-server.de";
/** Robot allows 200 requests an hour: a read asks about at most this many servers one by one. */
export const MAX_DETAIL = 25;
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
const arr = (v: unknown): unknown[] => (Array.isArray(v) ? v : []);
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : typeof v === "string" && v.trim() !== "" && Number.isFinite(Number(v)) ? Number(v) : null);
/** A date ("2026-12-01", or with a time) as "YYYY-MM-DD"; null for anything else. */
const day = (v: unknown): string | null => {
const s = str(v);
return s !== null && /^\d{4}-\d{2}-\d{2}/.test(s) && Number.isFinite(Date.parse(s.slice(0, 10))) ? s.slice(0, 10) : null;
};
/** Pure: one entry of `/server` or the answer of `/server/{number}` — wrapped in `{ server: {…} }`, or not. */
export function parseServer(raw: unknown): RobotServer | null {
const o = obj(raw);
const s = obj(o.server ?? raw);
const number = num(s.server_number);
if (number === null) return null;
const rescue = s.rescue;
return {
number,
name: str(s.server_name) ?? str(s.product) ?? `Server ${number}`,
ip: str(s.server_ip),
ipv6: str(s.server_ipv6_net),
product: str(s.product),
dc: str(s.dc),
traffic: str(s.traffic),
status: str(s.status)?.toLowerCase() ?? "unknown",
cancelled: s.cancelled === true,
paidUntil: day(s.paid_until),
rescue: typeof rescue === "boolean" ? rescue : null,
};
}
/** Pure: one entry of `/storagebox`, wrapped in `{ storagebox: {…} }` or not. */
export function parseStorageBox(raw: unknown): RobotStorageBox | null {
const o = obj(raw);
const s = obj(o.storagebox ?? raw);
const id = num(s.id);
if (id === null) return null;
return {
id,
name: str(s.name) ?? str(s.login) ?? `Storage box ${id}`,
login: str(s.login),
product: str(s.product),
location: str(s.location),
cancelled: s.cancelled === true,
locked: s.locked === true,
paidUntil: day(s.paid_until),
linkedServer: num(s.linked_server),
};
}
export function createHetznerRobotAdapter(config: HetznerRobotConfig): HetznerRobotAdapter {
const base = String(config.apiUrl ?? API).replace(/\/+$/, "");
const auth = `Basic ${Buffer.from(`${String(config.user ?? "").trim()}:${String(config.password ?? "")}`).toString("base64")}`;
/** Asks for something; a list that is empty is answered with a 404 and NOT_FOUND in its code, and `emptyOk` reads that as an empty list. */
async function get(path: string, emptyOk = false): Promise<unknown> {
const res = await requestJson(`${base}${path}`, { headers: { Authorization: auth, Accept: "application/json" } });
const error = obj(obj(res.json).error);
const code = str(error.code);
if (res.status === 401) throw new Error("Hetzner Robot did not accept the user name and password (HTTP 401) — use the webservice user of Robot → Settings → Webservice and app settings, not the login to Robot itself");
if (res.status === 403) throw new Error(`Hetzner Robot refused the request (HTTP 403${code ? `, ${code}` : ""}) — it blocks an address for a while after wrong passwords or more than 200 requests in an hour; wait, then check the webservice user`);
if (res.status === 404 && emptyOk && code !== null && /NOT_FOUND$/.test(code)) return [];
if (res.status === 404) throw new Error(`Hetzner Robot has no such address (HTTP 404${code ? `, ${code}` : ""})`);
if (res.status < 200 || res.status >= 300) throw new Error(`Hetzner Robot error: HTTP ${res.status}${code ? ` (${code})` : ""}${str(error.message) ? ` — ${str(error.message)}` : ""}`);
if (res.json === null) throw new Error("Hetzner Robot answered with something that isn't JSON");
return res.json;
}
async function read(): Promise<RobotRead> {
// The list of servers must work: it is what says the account is there.
const listed = arr(await get("/server", true)).map(parseServer).filter((s): s is RobotServer => s !== null);
const servers = [...listed].sort((a, b) => a.number - b.number);
const asked = servers.slice(0, MAX_DETAIL);
// The rescue system is not in the list: ask about each server, a few at a time.
for (let i = 0; i < asked.length; i += 4) {
await Promise.all(
asked.slice(i, i + 4).map(async (s) => {
try {
const detail = parseServer(await get(`/server/${s.number}`));
s.rescue = detail?.rescue ?? null;
} catch {
s.rescue = null;
}
}),
);
}
let storageBoxes: RobotStorageBox[] | null = null;
try {
storageBoxes = arr(await get("/storagebox", true)).map(parseStorageBox).filter((b): b is RobotStorageBox => b !== null).sort((a, b) => a.id - b.id);
} catch {
storageBoxes = null;
}
return { servers, storageBoxes, skippedDetail: Math.max(0, servers.length - asked.length) };
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
const start = Date.now();
try {
await get("/server", true);
return { ok: true, latencyMs: Date.now() - start };
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) };
}
}
return withDiagLogging("hetznerrobot", { ping, read });
}
+3
View File
@@ -22,6 +22,7 @@ import { createPaperlessAdapter } from "./paperless/adapter.js";
import { createN8nAdapter } from "./n8n/adapter.js";
import { createNetBoxAdapter } from "./netbox/adapter.js";
import { createHetznerAdapter } from "./hetzner/adapter.js";
import { createHetznerRobotAdapter } from "./hetznerRobot/adapter.js";
import { createSpeedtestAdapter } from "./speedtest/adapter.js";
import { createFragnetAdapter } from "./fragnet/adapter.js";
import { createArrAdapter } from "./arr/adapter.js";
@@ -88,6 +89,8 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
return createSpeedtestAdapter(config as any);
case "hetzner":
return createHetznerAdapter(config as any);
case "hetznerrobot":
return createHetznerRobotAdapter(config as any);
case "fragnet":
return createFragnetAdapter(config as any);
case "sonarr":
+43
View File
@@ -0,0 +1,43 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { hetznerRobotState, integrations } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { daysLeft, isRobotFresh, robotProblems, robotSyncStatuses, storedOf } from "../services/hetznerRobotSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const hetznerRobotRouter = Router();
hetznerRobotRouter.use(requireAuth);
/** Every Hetzner Robot account, how it is, from what the last reads stored: its dedicated servers (rescue system, cancellation), its storage boxes and what is wrong. Everyone signed in can see it. Nothing here changes anything at Hetzner. */
hetznerRobotRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, states, statuses] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "hetznerrobot")).orderBy(integrations.name),
db.select().from(hetznerRobotState),
robotSyncStatuses(),
]);
const byId = new Map(states.map((s) => [s.integrationId, s]));
const now = Date.now();
res.json({
accounts: rows.map((r) => {
const row = byId.get(r.id);
const state = row ? storedOf(row) : null;
const status = statuses[r.id] ?? null;
const fresh = isRobotFresh(status ?? undefined, now);
return {
id: r.id,
name: r.name,
enabled: r.enabled,
status,
fresh,
state: state && {
...state,
/** Calendar days to the end of each cancelled server and storage box: 0 on the day it ends, negative after. */
serverDays: Object.fromEntries(state.servers.filter((s) => s.cancelled).map((s) => [s.number, daysLeft(s.paidUntil, now)])),
boxDays: Object.fromEntries((state.storageBoxes ?? []).filter((b) => b.cancelled).map((b) => [b.id, daysLeft(b.paidUntil, now)])),
},
problems: state && fresh ? robotProblems(r.id, r.name, state, now) : [],
};
}),
});
}));
+4
View File
@@ -45,6 +45,7 @@ import { ARR_KINDS } from "../integrations/arr/adapter.js";
import { forgetAdguard, syncAdguard } from "../services/adguardSync.js";
import { forgetDownload, syncDownload } from "../services/downloadsSync.js";
import { DOWNLOAD_KINDS } from "../integrations/downloads/adapter.js";
import { forgetRobot, syncRobot } from "../services/hetznerRobotSync.js";
import { forgetTechnitium, syncTechnitium } from "../services/technitiumSync.js";
import { forgetGrafana, syncGrafana } from "../services/grafanaSync.js";
import { forgetNextcloud, syncNextcloud } from "../services/nextcloudSync.js";
@@ -163,6 +164,7 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
if (type === "fragnet") void syncFragnet(created.id).then(() => invalidateAlertsCache());
if ((ARR_KINDS as string[]).includes(type)) void syncArr(created.id).then(() => invalidateAlertsCache());
if ((DOWNLOAD_KINDS as string[]).includes(type)) void syncDownload(created.id).then(() => invalidateAlertsCache());
if (type === "hetznerrobot") void syncRobot(created.id).then(() => invalidateAlertsCache());
if (type === "adguard") void syncAdguard(created.id).then(() => invalidateAlertsCache());
if (type === "technitium") void syncTechnitium(created.id).then(() => invalidateAlertsCache());
if (type === "grafana") void syncGrafana(created.id).then(() => invalidateAlertsCache());
@@ -283,6 +285,7 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
if (updated.type === "fragnet" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncFragnet(updated.id).then(() => invalidateAlertsCache());
if ((ARR_KINDS as string[]).includes(updated.type) && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncArr(updated.id).then(() => invalidateAlertsCache());
if ((DOWNLOAD_KINDS as string[]).includes(updated.type) && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncDownload(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "hetznerrobot" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncRobot(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "adguard" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncAdguard(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "technitium" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncTechnitium(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "grafana" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncGrafana(updated.id).then(() => invalidateAlertsCache());
@@ -410,6 +413,7 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
if (existing.type === "fragnet") await forgetFragnet(id);
if ((ARR_KINDS as string[]).includes(existing.type)) await forgetArr(id);
if ((DOWNLOAD_KINDS as string[]).includes(existing.type)) await forgetDownload(id);
if (existing.type === "hetznerrobot") await forgetRobot(id);
if (existing.type === "adguard") await forgetAdguard(id);
if (existing.type === "technitium") await forgetTechnitium(id);
if (existing.type === "grafana") await forgetGrafana(id);
+4 -1
View File
@@ -91,6 +91,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
lidarr: { label: "Lidarr", link: "/media" },
readarr: { label: "Readarr", link: "/media" },
prowlarr: { label: "Prowlarr", link: "/media" },
hetznerrobot: { label: "Hetzner Robot", link: "/hetzner-robot" },
qbittorrent: { label: "qBittorrent", link: "/downloads" },
sabnzbd: { label: "SABnzbd", link: "/downloads" },
adguard: { label: "AdGuard Home", link: "/adguard" },
@@ -149,6 +150,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "grafana", id: Number(m[2]), category: m[1] === "ds" ? "integration" : "monitoring" };
m = /^(sonarr|radarr|prowlarr|lidarr|readarr)-(health|queue|disk):(\d+)/.exec(key);
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
m = /^hetznerrobot-(ending|rescue|setup|boxlocked|boxending):(\d+):/.exec(key);
if (m) return { type: "hetznerrobot", id: Number(m[2]), category: "services" };
m = /^(qbittorrent|sabnzbd)-(stuck|connection|paused|stalled|disk):(\d+)/.exec(key);
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
@@ -305,7 +308,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr|lidarr|readarr|qbittorrent|sabnzbd|adguard|technitium|grafana):(\d+)(?::(.+))?$/.exec(h);
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|hetznerrobot|speedtest|fragnet|sonarr|radarr|prowlarr|lidarr|readarr|qbittorrent|sabnzbd|adguard|technitium|grafana):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] === "images" ? " (age of the images)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
+25 -1
View File
@@ -1,6 +1,6 @@
import { and, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { downloadState, technitiumState, authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState , adguardState , grafanaState } from "../db/schema.js";
import { hetznerRobotState, downloadState, technitiumState, authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState , adguardState , grafanaState } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
@@ -40,6 +40,7 @@ import { isTechnitiumFresh, storedOf as technitiumStoredOf, technitiumProblems,
import { grafanaProblems, grafanaSyncStatuses, isGrafanaFresh, storedOf as grafanaStoredOf, type GrafanaProblem } from "./grafanaSync.js";
import { arrProblems, arrSyncStatuses, isArrFresh, storedOf as arrStoredOf, type ArrProblem } from "./arrSync.js";
import { DOWNLOAD_KINDS, type DownloadKind } from "../integrations/downloads/adapter.js";
import { isRobotFresh, robotProblems, robotSyncStatuses, storedOf as robotStoredOf, type RobotProblem } from "./hetznerRobotSync.js";
import { downloadProblems, downloadSyncStatuses, isDownloadFresh, storedOf as downloadStoredOf, type DownloadProblem } from "./downloadsSync.js";
import { fragnetProblems, fragnetSyncStatuses, isFragnetFresh, optionsOf as fragnetOptions, serversOf as fragnetServersOf, type FragnetProblem } from "./fragnetSync.js";
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
@@ -144,6 +145,8 @@ export interface HealthSnapshot {
arr?: { integrationId: number; kind: ArrKind; problems: ArrProblem[] }[];
/** What is wrong with each qBittorrent and SABnzbd (downloads that are stuck, no connection, a queue left paused or getting nothing, a disk nearly full), only while that reading is recent (services/downloadsSync.ts). */
downloads?: { integrationId: number; kind: DownloadKind; problems: DownloadProblem[] }[];
/** What is wrong with each Hetzner Robot account (a server or storage box about to shut down, a locked storage box, a rescue system left on, a server stuck being set up), only while that reading is recent (services/hetznerRobotSync.ts). */
hetznerRobot?: { integrationId: number; problems: RobotProblem[] }[];
/** What is wrong with each AdGuard Home (DNS not running or not answering, protection left off, filter lists not updating), only while that reading is recent (services/adguardSync.ts). */
adguard?: { integrationId: number; problems: AdguardProblem[] }[];
/** What is wrong with each Technitium DNS Server (DNS not answering, blocking left off, block lists not updating, zones expired or not syncing), only while that reading is recent (services/technitiumSync.ts). */
@@ -301,6 +304,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
for (const t of snapshot.technitium ?? []) for (const p of t.problems) out.push({ key: p.key, source: `technitium:${t.integrationId}`, message: p.message, severity: p.severity });
for (const g of snapshot.grafana ?? []) for (const p of g.problems) out.push({ key: p.key, source: `grafana:${g.integrationId}`, message: p.message, severity: p.severity });
// Hetzner Robot: servers and storage boxes about to shut down, a locked storage box, a rescue system left on, a server stuck being set up.
for (const r of snapshot.hetznerRobot ?? []) for (const p of r.problems) out.push({ key: p.key, source: `hetznerrobot:${r.integrationId}`, message: p.message, severity: p.severity });
// qBittorrent and SABnzbd: downloads that are stuck, no connection, a queue left paused or getting nothing, disks nearly full.
for (const d of snapshot.downloads ?? []) for (const p of d.problems) out.push({ key: p.key, source: `${d.kind}:${d.integrationId}`, message: p.message, severity: p.severity });
@@ -693,6 +699,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
console.error("[health] couldn't judge Sonarr, Radarr and Prowlarr:", err instanceof Error ? err.message : err);
}
// Hetzner Robot: read by its own pass every fifteen minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsH = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "hetznerrobot"), eq(integrations.enabled, true)));
if (rowsH.length > 0) {
const statuses = await robotSyncStatuses();
for (const r of rowsH) {
const [state] = await db.select().from(hetznerRobotState).where(eq(hetznerRobotState.integrationId, r.id)).limit(1);
if (!state || !isRobotFresh(statuses[r.id], Date.now())) {
held.add(`hetznerrobot:${r.id}`);
continue;
}
(snapshot.hetznerRobot ??= []).push({ integrationId: r.id, problems: robotProblems(r.id, r.name, robotStoredOf(state), Date.now()) });
}
}
} catch (err) {
console.error("[health] couldn't judge Hetzner Robot:", err instanceof Error ? err.message : err);
}
// qBittorrent and SABnzbd: read by their own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsD = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(and(inArray(integrations.type, DOWNLOAD_KINDS), eq(integrations.enabled, true)));
+210
View File
@@ -0,0 +1,210 @@
/**
* Keeps what Hetzner Robot says up to date: every fifteen minutes each Robot integration is read — its dedicated servers (and whether each one's rescue system is on) and its storage boxes — and the
* state is stored. Pages and alerts are built from what is stored. (Fifteen minutes, not five: Robot allows 200 requests an hour and a read asks about each server.)
*
* What is raised: **a server or storage box that is cancelled and about to shut down** (warning within 30 days, critical within 7 — the date Robot gives is the day it ends and the data goes with it);
* **a storage box that is locked** (critical: its data cannot be reached); **a rescue system left on** for two hours (the server's next boot is into the rescue system, not its own); **a server still
* "in process"** a day after the app first saw it so (an order that was never finished). "Since when" is what this app saw: the time of the first read that showed it.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { hetznerRobotState, integrations } from "../db/schema.js";
import { createHetznerRobotAdapter, type RobotRead, type RobotServer, type RobotStorageBox } from "../integrations/hetznerRobot/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 15 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 45 * 60_000;
const STATUS_FLAG = "hetznerRobotSync";
const MIN = 60_000;
const HOUR = 60 * MIN;
const DAY = 24 * HOUR;
/** A rescue system on for this long has been forgotten. */
export const RESCUE_AFTER_MS = 2 * HOUR;
/** A server that has not been "ready" for this long is not being set up any more, it is stuck. */
export const NOT_READY_AFTER_MS = DAY;
/** A cancelled server or storage box this close to its end is news; this close, urgent. */
export const ENDING_WARN_DAYS = 30;
export const ENDING_CRITICAL_DAYS = 7;
export interface RobotSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, RobotSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, RobotSyncStatus>) : {};
} catch {
return {};
}
}
export async function robotSyncStatuses(): Promise<Record<number, RobotSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isRobotFresh = (status: RobotSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof hetznerRobotState.$inferSelect;
function parse<T>(stored: string | null): T | null {
if (stored === null) return null;
try {
return JSON.parse(stored) as T;
} catch {
return null;
}
}
/** A server as stored: as Robot says it, plus what the app has seen between reads. */
export interface StoredRobotServer extends RobotServer {
/** When the first read that showed the rescue system on was made; null while it is off (or not known). */
rescueSince: string | null;
/** When the first read that showed it not "ready" was made; null while it is ready. */
notReadySince: string | null;
}
export interface StoredRobot {
servers: StoredRobotServer[];
storageBoxes: RobotStorageBox[] | null;
skippedDetail: number;
updatedAt: string;
}
export const storedOf = (row: StateRow): StoredRobot => ({
servers: (parse<StoredRobotServer[]>(row.servers) ?? []).map((s) => ({ ...s, rescueSince: s.rescueSince ?? null, notReadySince: s.notReadySince ?? null })),
storageBoxes: parse<RobotStorageBox[]>(row.storageBoxes),
skippedDetail: row.skippedDetail,
updatedAt: row.updatedAt,
});
/** Pure: the servers as read, with what was seen between reads carried over from the last read by server number. A thing first seen wrong starts its clock now. */
export function carrySince(servers: RobotServer[], previous: StoredRobotServer[] | null, now: Date): StoredRobotServer[] {
const at = now.toISOString();
const before = new Map((previous ?? []).map((s) => [s.number, s]));
return servers.map((s) => ({
...s,
rescueSince: s.rescue === true ? (before.get(s.number)?.rescueSince ?? at) : null,
notReadySince: s.status !== "ready" ? (before.get(s.number)?.notReadySince ?? at) : null,
}));
}
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
export async function storeRead(integrationId: number, r: RobotRead, now: Date = new Date()): Promise<void> {
const [previous] = await db.select().from(hetznerRobotState).where(eq(hetznerRobotState.integrationId, integrationId)).limit(1);
const values = {
servers: JSON.stringify(carrySince(r.servers, previous ? storedOf(previous).servers : null, now)),
storageBoxes: r.storageBoxes && JSON.stringify(r.storageBoxes),
skippedDetail: r.skippedDetail,
updatedAt: now.toISOString(),
};
await db.insert(hetznerRobotState).values({ integrationId, ...values }).onConflictDoUpdate({ target: hetznerRobotState.integrationId, set: values });
}
/** Reads one Robot account and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncRobot(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<RobotRead>): Promise<RobotSyncStatus> {
let status: RobotSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "hetznerrobot") throw new Error("not a Hetzner Robot integration");
const result = await (read ? read(loaded.config) : createHetznerRobotAdapter(loaded.config as never).read());
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[hetzner-robot] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[hetzner-robot] couldn't record the result:", err);
}
return status;
}
export async function syncAllRobot(now: Date = new Date()): Promise<Record<number, RobotSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "hetznerrobot"), eq(integrations.enabled, true)));
const out: Record<number, RobotSyncStatus> = {};
for (const r of rows) out[r.id] = await syncRobot(r.id, now);
return out;
}
export async function forgetRobot(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface RobotProblem {
key: string;
message: string;
severity: "warning" | "critical";
}
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / DAY), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
/** Pure: how many calendar days (UTC) there are from today to a date "YYYY-MM-DD": 0 on the day itself, which it still runs through, and negative once it has passed. */
export function daysLeft(date: string | null, nowMs: number): number | null {
if (date === null) return null;
const end = Date.parse(date);
if (!Number.isFinite(end)) return null;
const now = new Date(nowMs);
return Math.round((end - Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / DAY);
}
const endsIn = (days: number) => (days < 0 ? "has ended" : days === 0 ? "ends today" : days === 1 ? "ends tomorrow" : `ends in ${days} days`);
/** Pure: what is wrong with one Robot account, from what is stored for it. */
export function robotProblems(id: number, name: string, s: StoredRobot, nowMs: number): RobotProblem[] {
const out: RobotProblem[] = [];
for (const v of s.servers) {
const label = `${v.name} (server ${v.number})`;
if (v.cancelled) {
const left = daysLeft(v.paidUntil, nowMs);
if (left !== null && left <= ENDING_WARN_DAYS) {
out.push({ key: `hetznerrobot-ending:${id}:${v.number}`, severity: left <= ENDING_CRITICAL_DAYS ? "critical" : "warning", message: `${name}: ${label} is cancelled and ${endsIn(left)} (${v.paidUntil}) — the server and its data go with it` });
}
}
if (v.rescue === true && v.rescueSince !== null && nowMs - Date.parse(v.rescueSince) >= RESCUE_AFTER_MS) {
out.push({ key: `hetznerrobot-rescue:${id}:${v.number}`, severity: "warning", message: `${name}: the rescue system of ${label} has been switched on for ${ago(nowMs - Date.parse(v.rescueSince))} — its next boot is into the rescue system, not its own` });
}
if (v.status !== "ready" && v.notReadySince !== null && nowMs - Date.parse(v.notReadySince) >= NOT_READY_AFTER_MS) {
out.push({ key: `hetznerrobot-setup:${id}:${v.number}`, severity: "warning", message: `${name}: ${label} has been “${v.status}” for ${ago(nowMs - Date.parse(v.notReadySince))} instead of ready` });
}
}
for (const b of s.storageBoxes ?? []) {
const label = `storage box ${b.name}${b.login ? ` (${b.login})` : ""}`;
if (b.locked) out.push({ key: `hetznerrobot-boxlocked:${id}:${b.id}`, severity: "critical", message: `${name}: ${label} is locked — its data cannot be reached` });
if (b.cancelled) {
const left = daysLeft(b.paidUntil, nowMs);
if (left !== null && left <= ENDING_WARN_DAYS) {
out.push({ key: `hetznerrobot-boxending:${id}:${b.id}`, severity: left <= ENDING_CRITICAL_DAYS ? "critical" : "warning", message: `${name}: ${label} is cancelled and ${endsIn(left)} (${b.paidUntil}) — its data goes with it` });
}
}
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initHetznerRobotSync(): void {
if (timer) return;
const pass = () => void syncAllRobot().catch((err) => console.error("[hetzner-robot] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 47_000);
console.log(`[hetzner-robot] reading Hetzner Robot every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+1 -1
View File
@@ -3,7 +3,7 @@
* and not the other is caught. A kind in no group (an older one) is simply not in any group's summary.
*/
export const INTEGRATION_GROUPS: { id: string; label: string; types: string[] }[] = [
{ id: "infrastructure", label: "Infrastructure", types: ["proxmox", "synology", "dockhand", "hetzner", "fragnet", "tailscale"] },
{ id: "infrastructure", label: "Infrastructure", types: ["proxmox", "synology", "dockhand", "hetzner", "hetznerrobot", "fragnet", "tailscale"] },
{ id: "apps", label: "Apps", types: ["homeassistant", "jellyfin", "sonarr", "radarr", "lidarr", "readarr", "prowlarr", "qbittorrent", "sabnzbd", "immich", "paperless", "nextcloud", "authentik"] },
{ id: "backups", label: "Backups", types: ["pbs"] },
{ id: "network", label: "Network", types: ["unifi", "npm", "phpipam", "netbox", "adguard", "technitium"] },
+2
View File
@@ -45,6 +45,8 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `integration:${m[1]}`;
m = /^(?:sonarr|radarr|prowlarr|lidarr|readarr)-(?:health|queue|disk):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^hetznerrobot-(?:ending|rescue|setup|boxlocked|boxending):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^(?:qbittorrent|sabnzbd)-(?:stuck|connection|paused|stalled|disk):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^version-behind:(\d+)$/.exec(key);
+31 -1
View File
@@ -5,7 +5,7 @@
*/
import { eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { downloadState, technitiumState, assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState, adguardState, grafanaState } from "../db/schema.js";
import { hetznerRobotState, downloadState, technitiumState, assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState, adguardState, grafanaState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { backupState, listAll as listDuplicati } from "./duplicati.js";
@@ -15,6 +15,7 @@ import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf, loadBalancersOf as hetznerLoadBalancersOf, certificatesOf as hetznerCertificatesOf, trafficForecast as hetznerTrafficForecast } from "./hetznerSync.js";
import { ARR_KINDS } from "../integrations/arr/adapter.js";
import { daysLeft as robotDaysLeft, isRobotFresh, robotSyncStatuses, storedOf as robotStoredOf } from "./hetznerRobotSync.js";
import { DOWNLOAD_KINDS, DOWNLOAD_STATES } from "../integrations/downloads/adapter.js";
import { downloadSyncStatuses, isDownloadFresh, recentFailed as downloadsFailed, storedOf as downloadStoredOf, stuckItems as downloadStuck } from "./downloadsSync.js";
import { versionRows as appVersionRows } from "./appVersions.js";
@@ -212,6 +213,34 @@ async function addArrMetrics(out: Exposition, now: Date) {
}
}
/** Each Hetzner Robot account, for /metrics: for each dedicated server whether its rescue system is on and, if it is cancelled, the days left; for each storage box whether it is locked and, if cancelled, the days left. Only while the reading is recent. */
async function addRobotMetrics(out: Exposition, now: Date) {
const statuses = await robotSyncStatuses();
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "hetznerrobot"))).map((r) => [r.id, r]));
for (const s of await db.select().from(hetznerRobotState)) {
const row = rows.get(s.integrationId);
if (!row || !row.enabled || !isRobotFresh(statuses[s.integrationId], now.getTime())) continue;
const state = robotStoredOf(s);
out.add("homelab_hetzner_robot_servers", "gauge", "How many dedicated servers the Hetzner Robot account has.", state.servers.length, { name: row.name });
for (const v of state.servers) {
const labels = { name: row.name, server: String(v.number), server_name: v.name };
if (v.rescue !== null) out.add("homelab_hetzner_robot_rescue_active", "gauge", "1 if a dedicated server's rescue system is switched on for the next boot, else 0.", v.rescue ? 1 : 0, labels);
if (v.cancelled) {
const left = robotDaysLeft(v.paidUntil, now.getTime());
if (left !== null) out.add("homelab_hetzner_robot_days_left", "gauge", "Calendar days left of a cancelled dedicated server or storage box: 0 on the day it ends, negative after.", left, labels);
}
}
for (const b of state.storageBoxes ?? []) {
const labels = { name: row.name, storage_box: String(b.id), storage_box_name: b.name };
out.add("homelab_hetzner_robot_storage_box_locked", "gauge", "1 if a storage box is locked, else 0.", b.locked ? 1 : 0, labels);
if (b.cancelled) {
const left = robotDaysLeft(b.paidUntil, now.getTime());
if (left !== null) out.add("homelab_hetzner_robot_days_left", "gauge", "Calendar days left of a cancelled dedicated server or storage box: 0 on the day it ends, negative after.", left, labels);
}
}
}
}
/** Each qBittorrent and SABnzbd, for /metrics: how fast it is going, the items in each state and how many are stuck, whether it is connected or paused, what failed in the last day and the free space of its disk. Only while the reading is recent. */
async function addDownloadMetrics(out: Exposition, now: Date) {
const statuses = await downloadSyncStatuses();
@@ -609,6 +638,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addFragnetMetrics(out, now);
await addArrMetrics(out, now);
await addDownloadMetrics(out, now);
await addRobotMetrics(out, now);
await addAdguardMetrics(out, now);
await addTechnitiumMetrics(out, now);
await addVersionMetrics(out);