Read every 15 minutes (Robot allows 200 requests an hour; read-only): each server's product, addresses, status, rescue system and paid-until, and the storage boxes. Adapter, sync service, hetzner_robot_state table, /api/hetzner-robot, Infrastructure → Hetzner Robot page, metrics and docs. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
211 lines
11 KiB
TypeScript
211 lines
11 KiB
TypeScript
/**
|
|
* Keeps what Hetzner Robot says up to date: every fifteen minutes each Robot integration is read — its dedicated servers (and whether each one's rescue system is on) and its storage boxes — and the
|
|
* state is stored. Pages and alerts are built from what is stored. (Fifteen minutes, not five: Robot allows 200 requests an hour and a read asks about each server.)
|
|
*
|
|
* What is raised: **a server or storage box that is cancelled and about to shut down** (warning within 30 days, critical within 7 — the date Robot gives is the day it ends and the data goes with it);
|
|
* **a storage box that is locked** (critical: its data cannot be reached); **a rescue system left on** for two hours (the server's next boot is into the rescue system, not its own); **a server still
|
|
* "in process"** a day after the app first saw it so (an order that was never finished). "Since when" is what this app saw: the time of the first read that showed it.
|
|
*/
|
|
import { and, eq } from "drizzle-orm";
|
|
import { db } from "../db/client.js";
|
|
import { hetznerRobotState, integrations } from "../db/schema.js";
|
|
import { createHetznerRobotAdapter, type RobotRead, type RobotServer, type RobotStorageBox } from "../integrations/hetznerRobot/adapter.js";
|
|
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
|
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
|
|
|
export const SYNC_INTERVAL_MS = 15 * 60_000;
|
|
/** After this long without a good read, what is stored is no longer believed. */
|
|
export const STALE_AFTER_MS = 45 * 60_000;
|
|
const STATUS_FLAG = "hetznerRobotSync";
|
|
|
|
const MIN = 60_000;
|
|
const HOUR = 60 * MIN;
|
|
const DAY = 24 * HOUR;
|
|
/** A rescue system on for this long has been forgotten. */
|
|
export const RESCUE_AFTER_MS = 2 * HOUR;
|
|
/** A server that has not been "ready" for this long is not being set up any more, it is stuck. */
|
|
export const NOT_READY_AFTER_MS = DAY;
|
|
/** A cancelled server or storage box this close to its end is news; this close, urgent. */
|
|
export const ENDING_WARN_DAYS = 30;
|
|
export const ENDING_CRITICAL_DAYS = 7;
|
|
|
|
export interface RobotSyncStatus {
|
|
/** The last read that worked (kept when a later one failed). */
|
|
at: string;
|
|
ok: boolean;
|
|
error: string | null;
|
|
}
|
|
|
|
async function readStatuses(): Promise<Record<string, RobotSyncStatus>> {
|
|
try {
|
|
const raw = await getInternalFlag(STATUS_FLAG);
|
|
return raw ? (JSON.parse(raw) as Record<string, RobotSyncStatus>) : {};
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
export async function robotSyncStatuses(): Promise<Record<number, RobotSyncStatus>> {
|
|
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
|
}
|
|
|
|
export const isRobotFresh = (status: RobotSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
|
|
|
export type StateRow = typeof hetznerRobotState.$inferSelect;
|
|
|
|
function parse<T>(stored: string | null): T | null {
|
|
if (stored === null) return null;
|
|
try {
|
|
return JSON.parse(stored) as T;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** A server as stored: as Robot says it, plus what the app has seen between reads. */
|
|
export interface StoredRobotServer extends RobotServer {
|
|
/** When the first read that showed the rescue system on was made; null while it is off (or not known). */
|
|
rescueSince: string | null;
|
|
/** When the first read that showed it not "ready" was made; null while it is ready. */
|
|
notReadySince: string | null;
|
|
}
|
|
|
|
export interface StoredRobot {
|
|
servers: StoredRobotServer[];
|
|
storageBoxes: RobotStorageBox[] | null;
|
|
skippedDetail: number;
|
|
updatedAt: string;
|
|
}
|
|
|
|
export const storedOf = (row: StateRow): StoredRobot => ({
|
|
servers: (parse<StoredRobotServer[]>(row.servers) ?? []).map((s) => ({ ...s, rescueSince: s.rescueSince ?? null, notReadySince: s.notReadySince ?? null })),
|
|
storageBoxes: parse<RobotStorageBox[]>(row.storageBoxes),
|
|
skippedDetail: row.skippedDetail,
|
|
updatedAt: row.updatedAt,
|
|
});
|
|
|
|
/** Pure: the servers as read, with what was seen between reads carried over from the last read by server number. A thing first seen wrong starts its clock now. */
|
|
export function carrySince(servers: RobotServer[], previous: StoredRobotServer[] | null, now: Date): StoredRobotServer[] {
|
|
const at = now.toISOString();
|
|
const before = new Map((previous ?? []).map((s) => [s.number, s]));
|
|
return servers.map((s) => ({
|
|
...s,
|
|
rescueSince: s.rescue === true ? (before.get(s.number)?.rescueSince ?? at) : null,
|
|
notReadySince: s.status !== "ready" ? (before.get(s.number)?.notReadySince ?? at) : null,
|
|
}));
|
|
}
|
|
|
|
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
|
|
export async function storeRead(integrationId: number, r: RobotRead, now: Date = new Date()): Promise<void> {
|
|
const [previous] = await db.select().from(hetznerRobotState).where(eq(hetznerRobotState.integrationId, integrationId)).limit(1);
|
|
const values = {
|
|
servers: JSON.stringify(carrySince(r.servers, previous ? storedOf(previous).servers : null, now)),
|
|
storageBoxes: r.storageBoxes && JSON.stringify(r.storageBoxes),
|
|
skippedDetail: r.skippedDetail,
|
|
updatedAt: now.toISOString(),
|
|
};
|
|
await db.insert(hetznerRobotState).values({ integrationId, ...values }).onConflictDoUpdate({ target: hetznerRobotState.integrationId, set: values });
|
|
}
|
|
|
|
/** Reads one Robot account and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
|
export async function syncRobot(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<RobotRead>): Promise<RobotSyncStatus> {
|
|
let status: RobotSyncStatus;
|
|
try {
|
|
const loaded = await loadIntegrationConfig(integrationId);
|
|
if (!loaded || loaded.integration.type !== "hetznerrobot") throw new Error("not a Hetzner Robot integration");
|
|
const result = await (read ? read(loaded.config) : createHetznerRobotAdapter(loaded.config as never).read());
|
|
await storeRead(integrationId, result, now);
|
|
status = { at: now.toISOString(), ok: true, error: null };
|
|
} catch (err) {
|
|
const previous = (await readStatuses())[String(integrationId)];
|
|
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
|
console.error(`[hetzner-robot] couldn't read integration ${integrationId}:`, status.error);
|
|
}
|
|
try {
|
|
const all = await readStatuses();
|
|
all[String(integrationId)] = status;
|
|
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
|
} catch (err) {
|
|
console.error("[hetzner-robot] couldn't record the result:", err);
|
|
}
|
|
return status;
|
|
}
|
|
|
|
export async function syncAllRobot(now: Date = new Date()): Promise<Record<number, RobotSyncStatus>> {
|
|
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "hetznerrobot"), eq(integrations.enabled, true)));
|
|
const out: Record<number, RobotSyncStatus> = {};
|
|
for (const r of rows) out[r.id] = await syncRobot(r.id, now);
|
|
return out;
|
|
}
|
|
|
|
export async function forgetRobot(integrationId: number): Promise<void> {
|
|
const all = await readStatuses();
|
|
if (String(integrationId) in all) {
|
|
delete all[String(integrationId)];
|
|
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
|
}
|
|
}
|
|
|
|
// ─── What is wrong ──────────────────────────────────────────────────────────
|
|
|
|
export interface RobotProblem {
|
|
key: string;
|
|
message: string;
|
|
severity: "warning" | "critical";
|
|
}
|
|
|
|
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
|
|
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / DAY), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
|
|
|
|
/** Pure: how many calendar days (UTC) there are from today to a date "YYYY-MM-DD": 0 on the day itself, which it still runs through, and negative once it has passed. */
|
|
export function daysLeft(date: string | null, nowMs: number): number | null {
|
|
if (date === null) return null;
|
|
const end = Date.parse(date);
|
|
if (!Number.isFinite(end)) return null;
|
|
const now = new Date(nowMs);
|
|
return Math.round((end - Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / DAY);
|
|
}
|
|
|
|
const endsIn = (days: number) => (days < 0 ? "has ended" : days === 0 ? "ends today" : days === 1 ? "ends tomorrow" : `ends in ${days} days`);
|
|
|
|
/** Pure: what is wrong with one Robot account, from what is stored for it. */
|
|
export function robotProblems(id: number, name: string, s: StoredRobot, nowMs: number): RobotProblem[] {
|
|
const out: RobotProblem[] = [];
|
|
for (const v of s.servers) {
|
|
const label = `${v.name} (server ${v.number})`;
|
|
if (v.cancelled) {
|
|
const left = daysLeft(v.paidUntil, nowMs);
|
|
if (left !== null && left <= ENDING_WARN_DAYS) {
|
|
out.push({ key: `hetznerrobot-ending:${id}:${v.number}`, severity: left <= ENDING_CRITICAL_DAYS ? "critical" : "warning", message: `${name}: ${label} is cancelled and ${endsIn(left)} (${v.paidUntil}) — the server and its data go with it` });
|
|
}
|
|
}
|
|
if (v.rescue === true && v.rescueSince !== null && nowMs - Date.parse(v.rescueSince) >= RESCUE_AFTER_MS) {
|
|
out.push({ key: `hetznerrobot-rescue:${id}:${v.number}`, severity: "warning", message: `${name}: the rescue system of ${label} has been switched on for ${ago(nowMs - Date.parse(v.rescueSince))} — its next boot is into the rescue system, not its own` });
|
|
}
|
|
if (v.status !== "ready" && v.notReadySince !== null && nowMs - Date.parse(v.notReadySince) >= NOT_READY_AFTER_MS) {
|
|
out.push({ key: `hetznerrobot-setup:${id}:${v.number}`, severity: "warning", message: `${name}: ${label} has been “${v.status}” for ${ago(nowMs - Date.parse(v.notReadySince))} instead of ready` });
|
|
}
|
|
}
|
|
for (const b of s.storageBoxes ?? []) {
|
|
const label = `storage box ${b.name}${b.login ? ` (${b.login})` : ""}`;
|
|
if (b.locked) out.push({ key: `hetznerrobot-boxlocked:${id}:${b.id}`, severity: "critical", message: `${name}: ${label} is locked — its data cannot be reached` });
|
|
if (b.cancelled) {
|
|
const left = daysLeft(b.paidUntil, nowMs);
|
|
if (left !== null && left <= ENDING_WARN_DAYS) {
|
|
out.push({ key: `hetznerrobot-boxending:${id}:${b.id}`, severity: left <= ENDING_CRITICAL_DAYS ? "critical" : "warning", message: `${name}: ${label} is cancelled and ${endsIn(left)} (${b.paidUntil}) — its data goes with it` });
|
|
}
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
let timer: ReturnType<typeof setInterval> | null = null;
|
|
|
|
export function initHetznerRobotSync(): void {
|
|
if (timer) return;
|
|
const pass = () => void syncAllRobot().catch((err) => console.error("[hetzner-robot] pass failed:", err));
|
|
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
|
setTimeout(pass, 47_000);
|
|
console.log(`[hetzner-robot] reading Hetzner Robot every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
|
}
|