Hetzner Cloud integration: servers, traffic quota, backups, volumes and cost
One integration per project (read-only API token, read every 5 minutes). Alerts for a server that was running and is off, stuck statuses, traffic at 80%/100% of the monthly quota, rescue mode left on, and late backups. The Hetzner page lists servers, volumes, snapshots and a cost estimate; a server the app knows by its agent gets a Hetzner card on its own page. /metrics and docs updated. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5a2b320175
commit
24b708abbe
35 files changed
+9430
-7
No files matched your search
@@ -0,0 +1,9 @@
|
||||
CREATE TABLE `hetzner_state` (
|
||||
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||
`servers` text DEFAULT '[]' NOT NULL,
|
||||
`volumes` text,
|
||||
`images` text,
|
||||
`cost` text,
|
||||
`updated_at` text NOT NULL,
|
||||
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -477,6 +477,13 @@
|
||||
"when": 1792963314439,
|
||||
"tag": "0067_shallow_mastermind",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 68,
|
||||
"version": "6",
|
||||
"when": 1792964323718,
|
||||
"tag": "0068_glorious_steel_serpent",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -66,6 +66,7 @@ import { immichRouter } from "./routes/immich.js";
|
||||
import { paperlessRouter } from "./routes/paperless.js";
|
||||
import { n8nRouter } from "./routes/n8n.js";
|
||||
import { netboxRouter } from "./routes/netbox.js";
|
||||
import { hetznerRouter } from "./routes/hetzner.js";
|
||||
import { nextcloudRouter } from "./routes/nextcloud.js";
|
||||
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
|
||||
import { publicApiRouter } from "./routes/publicApi.js";
|
||||
@@ -209,6 +210,7 @@ export function createApp() {
|
||||
app.use("/api/paperless", paperlessRouter);
|
||||
app.use("/api/n8n", n8nRouter);
|
||||
app.use("/api/netbox", netboxRouter);
|
||||
app.use("/api/hetzner", hetznerRouter);
|
||||
app.use("/api/nextcloud", nextcloudRouter);
|
||||
app.use("/api/service-levels", serviceLevelsRouter);
|
||||
app.use("/api/todos", todosRouter);
|
||||
|
||||
@@ -561,6 +561,7 @@ export const integrationTypes = [
|
||||
"nextcloud",
|
||||
"netbox",
|
||||
"n8n",
|
||||
"hetzner",
|
||||
] as const;
|
||||
export type IntegrationType = (typeof integrationTypes)[number];
|
||||
|
||||
@@ -1218,6 +1219,21 @@ export const nextcloudState = sqliteTable("nextcloud_state", {
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── Hetzner Cloud ──────────────────────────────────────────────────────────
|
||||
|
||||
// How a Hetzner Cloud project (an integration of type "hetzner"; a token belongs to one project) is right now, one row each, replaced at every read: its servers, volumes, snapshots and
|
||||
// backups as JSON, and a cost estimate. A column that is null could not be read — which is not the same as none. Read by services/hetznerSync.ts.
|
||||
export const hetznerState = sqliteTable("hetzner_state", {
|
||||
integrationId: integer("integration_id")
|
||||
.primaryKey()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
servers: text("servers").notNull().default("[]"), // JSON: each server as read, plus when it was last seen running, since when it has had this status and since when rescue mode has been on
|
||||
volumes: text("volumes"), // JSON; null: could not be asked
|
||||
images: text("images"), // JSON: snapshots and backups; null: could not be asked
|
||||
cost: text("cost"), // JSON: the monthly estimate (currency, servers, backups, volumes, snapshots, IPs, total); null: could not be worked out (no price list)
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── NetBox ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// How a NetBox (an integration of type "netbox") is right now, one row each, replaced at every read: its version, how many background workers are running, the counts and the IP
|
||||
|
||||
@@ -38,6 +38,7 @@ import { initN8nSync } from "./services/n8nSync.js";
|
||||
import { initNetBoxSync } from "./services/netboxSync.js";
|
||||
import { initNextcloudSync } from "./services/nextcloudSync.js";
|
||||
import { initDockerVulnSync } from "./services/dockerVulnSync.js";
|
||||
import { initHetznerSync } from "./services/hetznerSync.js";
|
||||
import { backfillMetricRollups } from "./services/serverMetrics.js";
|
||||
import { backfillEndpointHourly } from "./services/endpointChecks.js";
|
||||
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
|
||||
@@ -97,6 +98,7 @@ initN8nSync();
|
||||
initNetBoxSync();
|
||||
initNextcloudSync();
|
||||
initDockerVulnSync();
|
||||
initHetznerSync();
|
||||
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
|
||||
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
|
||||
|
||||
|
||||
@@ -105,6 +105,7 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
|
||||
{ key: "appPassword", label: "App password (optional)", secret: true, optional: true, type: "password", placeholder: "Personal settings → Security → Devices & sessions" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
hetzner: [{ key: "token", label: "API token", secret: true, type: "password", placeholder: "Hetzner Console → the project → Security → API tokens; Read is enough" }],
|
||||
netbox: [
|
||||
{ key: "url", label: "NetBox URL", secret: false, placeholder: "https://netbox.example.lan" },
|
||||
{ key: "token", label: "API token", secret: true, type: "password", placeholder: "your profile → API Tokens; read-only is enough" },
|
||||
@@ -179,6 +180,7 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
|
||||
/** Fixed base URL per integration type, stored on the row for display/reference. */
|
||||
export const INTEGRATION_BASE_URLS: Partial<Record<IntegrationType, string>> = {
|
||||
tailscale: "https://api.tailscale.com",
|
||||
hetzner: "https://api.hetzner.cloud",
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
/**
|
||||
* Hetzner Cloud adapter. Config: token (a project's API token — Hetzner Console → the project → Security → API tokens; **Read** is enough).
|
||||
*
|
||||
* Reading only: the project's servers (status, type, location, addresses, traffic used and included this month, whether rescue mode is on and whether backups are enabled), its volumes, its
|
||||
* snapshots and backups, its primary IPs and Hetzner's price list — enough to say what is off, what is close to its traffic quota, what has no recent backup, and roughly what the project
|
||||
* costs. Nothing is sent to Hetzner but GET requests for these; no server is started, stopped or changed.
|
||||
*
|
||||
* Endpoints and field names are from Hetzner's published OpenAPI file (`/servers`, `/volumes`, `/images`, `/primary_ips`, `/pricing`; lists are paged, 50 at a time) and have not been
|
||||
* checked against a live project here; every field is read defensively. A token belongs to one project, so one integration is one project. The API allows 3600 requests an hour per project.
|
||||
*/
|
||||
import { requestJson } from "../httpJson.js";
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
|
||||
export interface HetznerConfig {
|
||||
token: string;
|
||||
/** Only for tests: the form never asks for it (and a request cannot set it — see fieldSchemas.ts). */
|
||||
apiUrl?: string;
|
||||
}
|
||||
|
||||
export interface HetznerServer {
|
||||
id: number;
|
||||
name: string;
|
||||
/** running, off, initializing, starting, stopping, migrating, rebuilding, deleting or unknown. */
|
||||
status: string;
|
||||
created: string | null;
|
||||
type: string | null;
|
||||
cores: number | null;
|
||||
memoryGb: number | null;
|
||||
diskGb: number | null;
|
||||
location: string | null;
|
||||
city: string | null;
|
||||
os: string | null;
|
||||
ipv4: string | null;
|
||||
ipv6: string | null;
|
||||
privateIps: string[];
|
||||
locked: boolean;
|
||||
rescueEnabled: boolean;
|
||||
/** Set when backups are enabled for the server (the hour they are taken in); null when they are not. */
|
||||
backupWindow: string | null;
|
||||
/** This month's traffic, in bytes; null when Hetzner did not say. */
|
||||
includedTraffic: number | null;
|
||||
outgoingTraffic: number | null;
|
||||
ingoingTraffic: number | null;
|
||||
/** The price of the server type in this location per month, including VAT, in the project's currency (read from the type's own price list). */
|
||||
monthlyPrice: number | null;
|
||||
}
|
||||
|
||||
export interface HetznerVolume {
|
||||
id: number;
|
||||
name: string;
|
||||
/** GB. */
|
||||
size: number;
|
||||
/** The server it is attached to; null when it is attached to none (it is billed all the same). */
|
||||
serverId: number | null;
|
||||
location: string | null;
|
||||
status: string;
|
||||
}
|
||||
|
||||
export interface HetznerImage {
|
||||
id: number;
|
||||
type: "snapshot" | "backup";
|
||||
description: string | null;
|
||||
created: string | null;
|
||||
/** GB of storage it takes (what is billed for a snapshot); null when it is still being made. */
|
||||
sizeGb: number | null;
|
||||
/** The server a backup belongs to (bound_to), or the one a snapshot was taken from (created_from) — which may be gone. */
|
||||
serverId: number | null;
|
||||
}
|
||||
|
||||
export interface HetznerPrimaryIp {
|
||||
id: number;
|
||||
ip: string;
|
||||
type: "ipv4" | "ipv6";
|
||||
location: string | null;
|
||||
assigned: boolean;
|
||||
}
|
||||
|
||||
export interface HetznerPrices {
|
||||
currency: string;
|
||||
/** Per GB and month, including VAT. */
|
||||
imagePerGb: number | null;
|
||||
volumePerGb: number | null;
|
||||
/** What a server's backups cost as a percentage of the server's price (20 at the time of writing). */
|
||||
backupPercent: number | null;
|
||||
/** Per month, including VAT, for each kind and location of primary IP. */
|
||||
primaryIps: { type: string; location: string; monthly: number }[];
|
||||
}
|
||||
|
||||
export interface HetznerRead {
|
||||
servers: HetznerServer[];
|
||||
/** Null for each of these when it could not be asked, which is not the same as none. */
|
||||
volumes: HetznerVolume[] | null;
|
||||
images: HetznerImage[] | null;
|
||||
primaryIps: HetznerPrimaryIp[] | null;
|
||||
prices: HetznerPrices | null;
|
||||
}
|
||||
|
||||
export interface HetznerAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(): Promise<HetznerRead>;
|
||||
}
|
||||
|
||||
const PER_PAGE = 50;
|
||||
export const MAX_PAGES = 20;
|
||||
const API = "https://api.hetzner.cloud/v1";
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
|
||||
const arr = (v: unknown): unknown[] => (Array.isArray(v) ? v : []);
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
|
||||
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||||
/** Hetzner sends prices as strings ("4.5100000000"). */
|
||||
const price = (v: unknown): number | null => {
|
||||
const n = typeof v === "string" ? Number(v) : v;
|
||||
return typeof n === "number" && Number.isFinite(n) && n >= 0 ? n : null;
|
||||
};
|
||||
|
||||
/** Pure: the page after this one, or null when this is the last. */
|
||||
export function nextPage(raw: unknown): number | null {
|
||||
return num(obj(obj(obj(raw).meta).pagination).next_page);
|
||||
}
|
||||
|
||||
/** Pure: the servers of one page of /servers. */
|
||||
export function parseServers(raw: unknown): HetznerServer[] {
|
||||
const out: HetznerServer[] = [];
|
||||
for (const item of arr(obj(raw).servers)) {
|
||||
const s = obj(item);
|
||||
const id = num(s.id);
|
||||
const name = str(s.name);
|
||||
if (id === null || !name) continue;
|
||||
const type = obj(s.server_type);
|
||||
const location = obj(s.location);
|
||||
const net = obj(s.public_net);
|
||||
const locationName = str(location.name);
|
||||
const priced = arr(type.prices).map(obj).find((p) => str(p.location) === locationName);
|
||||
out.push({
|
||||
id,
|
||||
name,
|
||||
status: str(s.status) ?? "unknown",
|
||||
created: str(s.created),
|
||||
type: str(type.name),
|
||||
cores: num(type.cores),
|
||||
memoryGb: num(type.memory),
|
||||
diskGb: num(type.disk) ?? num(s.primary_disk_size),
|
||||
location: locationName,
|
||||
city: str(location.city),
|
||||
os: str(obj(s.image).description) ?? str(obj(s.image).name),
|
||||
ipv4: str(obj(net.ipv4).ip),
|
||||
ipv6: str(obj(net.ipv6).ip),
|
||||
privateIps: arr(s.private_net).map((n) => str(obj(n).ip)).filter((ip): ip is string => ip !== null),
|
||||
locked: s.locked === true,
|
||||
rescueEnabled: s.rescue_enabled === true,
|
||||
backupWindow: str(s.backup_window),
|
||||
includedTraffic: num(s.included_traffic),
|
||||
outgoingTraffic: num(s.outgoing_traffic),
|
||||
ingoingTraffic: num(s.ingoing_traffic),
|
||||
monthlyPrice: price(obj(obj(priced).price_monthly).gross),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: the volumes of one page of /volumes. */
|
||||
export function parseVolumes(raw: unknown): HetznerVolume[] {
|
||||
const out: HetznerVolume[] = [];
|
||||
for (const item of arr(obj(raw).volumes)) {
|
||||
const v = obj(item);
|
||||
const id = num(v.id);
|
||||
const name = str(v.name);
|
||||
const size = num(v.size);
|
||||
if (id === null || !name || size === null) continue;
|
||||
out.push({ id, name, size, serverId: num(v.server), location: str(obj(v.location).name), status: str(v.status) ?? "unknown" });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: the snapshots and backups of one page of /images (anything else — Hetzner's own system images — is left out). */
|
||||
export function parseImages(raw: unknown): HetznerImage[] {
|
||||
const out: HetznerImage[] = [];
|
||||
for (const item of arr(obj(raw).images)) {
|
||||
const i = obj(item);
|
||||
const id = num(i.id);
|
||||
const type = i.type === "snapshot" || i.type === "backup" ? i.type : null;
|
||||
if (id === null || !type) continue;
|
||||
out.push({ id, type, description: str(i.description), created: str(i.created), sizeGb: num(i.image_size), serverId: type === "backup" ? num(i.bound_to) : num(obj(i.created_from).id) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: the primary IPs of one page of /primary_ips. */
|
||||
export function parsePrimaryIps(raw: unknown): HetznerPrimaryIp[] {
|
||||
const out: HetznerPrimaryIp[] = [];
|
||||
for (const item of arr(obj(raw).primary_ips)) {
|
||||
const p = obj(item);
|
||||
const id = num(p.id);
|
||||
const ip = str(p.ip);
|
||||
if (id === null || !ip || (p.type !== "ipv4" && p.type !== "ipv6")) continue;
|
||||
out.push({ id, ip, type: p.type, location: str(obj(p.location).name), assigned: num(p.assignee_id) !== null });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: what /pricing says that the cost estimate needs; null when it is not Hetzner's price list. */
|
||||
export function parsePrices(raw: unknown): HetznerPrices | null {
|
||||
const p = obj(obj(raw).pricing);
|
||||
const currency = str(p.currency);
|
||||
if (!currency) return null;
|
||||
const primaryIps: HetznerPrices["primaryIps"] = [];
|
||||
for (const kind of arr(p.primary_ips)) {
|
||||
const k = obj(kind);
|
||||
for (const entry of arr(k.prices)) {
|
||||
const monthly = price(obj(obj(entry).price_monthly).gross);
|
||||
const location = str(obj(entry).location);
|
||||
const type = str(k.type);
|
||||
if (monthly !== null && location && type) primaryIps.push({ type, location, monthly });
|
||||
}
|
||||
}
|
||||
return {
|
||||
currency,
|
||||
imagePerGb: price(obj(obj(p.image).price_per_gb_month).gross),
|
||||
volumePerGb: price(obj(obj(p.volume).price_per_gb_month).gross),
|
||||
backupPercent: price(obj(p.server_backup).percentage),
|
||||
primaryIps,
|
||||
};
|
||||
}
|
||||
|
||||
export function createHetznerAdapter(config: HetznerConfig): HetznerAdapter {
|
||||
const base = String(config.apiUrl ?? API).replace(/\/+$/, "");
|
||||
const headers = { Authorization: `Bearer ${config.token}`, Accept: "application/json" };
|
||||
|
||||
async function get(path: string): Promise<unknown> {
|
||||
const res = await requestJson(`${base}${path}`, { headers });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Hetzner did not accept the token (HTTP ${res.status}) — check it is a project's API token, and has not been deleted`);
|
||||
if (res.status === 429) throw new Error("Hetzner says too many requests (HTTP 429) — the project's hourly limit is used up; it recovers gradually");
|
||||
if (res.status < 200 || res.status >= 300) throw new Error(`Hetzner error: HTTP ${res.status}`);
|
||||
if (res.json === null) throw new Error("Hetzner answered with something that isn't JSON");
|
||||
return res.json;
|
||||
}
|
||||
|
||||
/** Every page of a list, up to MAX_PAGES. */
|
||||
async function all<T>(path: string, parse: (raw: unknown) => T[]): Promise<T[]> {
|
||||
const out: T[] = [];
|
||||
const sep = path.includes("?") ? "&" : "?";
|
||||
for (let page: number | null = 1, n = 0; page !== null && n < MAX_PAGES; n++) {
|
||||
const raw = await get(`${path}${sep}per_page=${PER_PAGE}&page=${page}`);
|
||||
out.push(...parse(raw));
|
||||
page = nextPage(raw);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const soft = async <T>(run: () => Promise<T>): Promise<T | null> => {
|
||||
try {
|
||||
return await run();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
async function read(): Promise<HetznerRead> {
|
||||
const servers = await all("/servers", parseServers);
|
||||
const [volumes, snapshots, backups, primaryIps, prices] = await Promise.all([
|
||||
soft(() => all("/volumes", parseVolumes)),
|
||||
soft(() => all("/images?type=snapshot", parseImages)),
|
||||
soft(() => all("/images?type=backup", parseImages)),
|
||||
soft(() => all("/primary_ips", parsePrimaryIps)),
|
||||
soft(async () => parsePrices(await get("/pricing"))),
|
||||
]);
|
||||
return { servers, volumes, images: snapshots && backups ? [...snapshots, ...backups] : null, primaryIps, prices };
|
||||
}
|
||||
|
||||
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
await get("/servers?per_page=1");
|
||||
return { ok: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
return withDiagLogging("hetzner", { ping, read });
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import { createImmichAdapter } from "./immich/adapter.js";
|
||||
import { createPaperlessAdapter } from "./paperless/adapter.js";
|
||||
import { createN8nAdapter } from "./n8n/adapter.js";
|
||||
import { createNetBoxAdapter } from "./netbox/adapter.js";
|
||||
import { createHetznerAdapter } from "./hetzner/adapter.js";
|
||||
import { createNextcloudAdapter } from "./nextcloud/adapter.js";
|
||||
|
||||
export interface PingableAdapter {
|
||||
@@ -76,6 +77,8 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
|
||||
return createPaperlessAdapter(config as any);
|
||||
case "n8n":
|
||||
return createN8nAdapter(config as any);
|
||||
case "hetzner":
|
||||
return createHetznerAdapter(config as any);
|
||||
case "netbox":
|
||||
return createNetBoxAdapter(config as any);
|
||||
case "nextcloud":
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { hetznerState, integrations, servers } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { costOf, hetznerProblems, hetznerSyncStatuses, imagesOf, isHetznerFresh, matchAppServer, serversOf, volumesOf } from "../services/hetznerSync.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const hetznerRouter = Router();
|
||||
hetznerRouter.use(requireAuth);
|
||||
|
||||
const parseIps = (stored: string | null): string[] => {
|
||||
if (!stored) return [];
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
const appServers = async () => (await db.select({ id: servers.id, name: servers.name, ips: servers.ipAddresses }).from(servers)).map((s) => ({ id: s.id, name: s.name, ips: parseIps(s.ips) }));
|
||||
|
||||
/** Every Hetzner project, how it is, from what the last read stored: its servers (and which of them the app also knows by its agent), volumes, snapshots, cost and what is wrong. Everyone signed in can see it. */
|
||||
hetznerRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, states, statuses, known, settings] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "hetzner")).orderBy(integrations.name),
|
||||
db.select().from(hetznerState),
|
||||
hetznerSyncStatuses(),
|
||||
appServers(),
|
||||
getSettings(),
|
||||
]);
|
||||
const byId = new Map(states.map((s) => [s.integrationId, s]));
|
||||
const now = Date.now();
|
||||
res.json({
|
||||
projects: rows.map((r) => {
|
||||
const state = byId.get(r.id) ?? null;
|
||||
const status = statuses[r.id] ?? null;
|
||||
const fresh = isHetznerFresh(status ?? undefined, now);
|
||||
const stored = state ? serversOf(state) : [];
|
||||
const images = state ? imagesOf(state) : null;
|
||||
const names = new Map(stored.map((s) => [s.id, s.name]));
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
enabled: r.enabled,
|
||||
status,
|
||||
fresh,
|
||||
state: state
|
||||
? {
|
||||
servers: stored.map((s) => ({ ...s, linked: matchAppServer(s, known) })),
|
||||
/** Null when the volumes could not be listed. */
|
||||
volumes: volumesOf(state)?.map((v) => ({ ...v, serverName: v.serverId === null ? null : (names.get(v.serverId) ?? null) })) ?? null,
|
||||
images: images?.map((i) => ({ ...i, serverName: i.serverId === null ? null : (names.get(i.serverId) ?? null) })) ?? null,
|
||||
cost: costOf(state),
|
||||
updatedAt: state.updatedAt,
|
||||
}
|
||||
: null,
|
||||
problems: state && fresh ? hetznerProblems(r.id, r.name, stored, images, now, settings.healthChecks.backupStaleHours) : [],
|
||||
};
|
||||
}),
|
||||
});
|
||||
}));
|
||||
|
||||
/** The Hetzner server that is this server (by address, else by name), for the card on its page; null when none is. */
|
||||
hetznerRouter.get("/for-server/:id", asyncHandler(async (req, res) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [self] = (await appServers()).filter((s) => s.id === id);
|
||||
if (!self) return res.status(404).json({ error: "not_found" });
|
||||
const [rows, states, statuses, settings] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(eq(integrations.type, "hetzner")),
|
||||
db.select().from(hetznerState),
|
||||
hetznerSyncStatuses(),
|
||||
getSettings(),
|
||||
]);
|
||||
const now = Date.now();
|
||||
for (const r of rows) {
|
||||
const state = states.find((s) => s.integrationId === r.id);
|
||||
if (!state) continue;
|
||||
const stored = serversOf(state);
|
||||
const server = stored.find((s) => matchAppServer(s, [self]) !== null);
|
||||
if (!server) continue;
|
||||
const fresh = isHetznerFresh(statuses[r.id], now);
|
||||
const images = imagesOf(state);
|
||||
const newestBackup = (images ?? []).filter((i) => i.type === "backup" && i.serverId === server.id && i.created).map((i) => i.created as string).sort().pop() ?? null;
|
||||
return res.json({
|
||||
match: {
|
||||
integrationId: r.id,
|
||||
integrationName: r.name,
|
||||
fresh,
|
||||
updatedAt: state.updatedAt,
|
||||
server,
|
||||
newestBackup,
|
||||
snapshots: (images ?? []).filter((i) => i.type === "snapshot" && i.serverId === server.id).length,
|
||||
problems: fresh ? hetznerProblems(r.id, r.name, stored, images, now, settings.healthChecks.backupStaleHours).filter((p) => p.serverId === server.id) : [],
|
||||
},
|
||||
});
|
||||
}
|
||||
res.json({ match: null });
|
||||
}));
|
||||
@@ -36,6 +36,7 @@ import { forgetImmich, syncImmich } from "../services/immichSync.js";
|
||||
import { forgetPaperless, syncPaperless } from "../services/paperlessSync.js";
|
||||
import { forgetN8n, syncN8n } from "../services/n8nSync.js";
|
||||
import { forgetNetBox, syncNetBox } from "../services/netboxSync.js";
|
||||
import { forgetHetzner, syncHetzner } from "../services/hetznerSync.js";
|
||||
import { forgetNextcloud, syncNextcloud } from "../services/nextcloudSync.js";
|
||||
import { dockerVulnSyncStatuses, findingsOf, forgetDockerVulns, isDockerVulnFresh, syncDockerVulns } from "../services/dockerVulnSync.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
@@ -147,6 +148,7 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
|
||||
if (type === "paperless") void syncPaperless(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "n8n") void syncN8n(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "netbox") void syncNetBox(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "hetzner") void syncHetzner(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "nextcloud") void syncNextcloud(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "dockhand") void syncDockerVulns(created.id).then(() => invalidateAlertsCache());
|
||||
|
||||
@@ -259,6 +261,7 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
if (updated.type === "paperless" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncPaperless(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "n8n" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncN8n(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "netbox" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncNetBox(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "hetzner" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncHetzner(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "nextcloud" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncNextcloud(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "dockhand" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncDockerVulns(updated.id).then(() => invalidateAlertsCache());
|
||||
|
||||
@@ -347,6 +350,7 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
if (existing.type === "paperless") await forgetPaperless(id);
|
||||
if (existing.type === "n8n") await forgetN8n(id);
|
||||
if (existing.type === "netbox") await forgetNetBox(id);
|
||||
if (existing.type === "hetzner") await forgetHetzner(id);
|
||||
if (existing.type === "nextcloud") await forgetNextcloud(id);
|
||||
if (existing.type === "dockhand") await forgetDockerVulns(id);
|
||||
await db.delete(integrations).where(eq(integrations.id, id));
|
||||
|
||||
@@ -83,6 +83,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
n8n: { label: "n8n", link: "/n8n" },
|
||||
netbox: { label: "NetBox", link: "/netbox" },
|
||||
nextcloud: { label: "Nextcloud", link: "/nextcloud" },
|
||||
hetzner: { label: "Hetzner", link: "/hetzner" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -121,6 +122,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^docker-vuln:(\d+):/.exec(key);
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^hetzner-(off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
|
||||
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
||||
@@ -261,7 +264,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
@@ -271,7 +274,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities } from "../db/schema.js";
|
||||
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -29,6 +29,7 @@ import { isN8nFresh, n8nProblems, n8nSyncStatuses, type N8nProblem } from "./n8n
|
||||
import { isNetBoxFresh, netboxProblems, netboxSyncStatuses, type NetBoxProblem } from "./netboxSync.js";
|
||||
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type NextcloudProblem } from "./nextcloudSync.js";
|
||||
import { dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, type DockerVulnProblem } from "./dockerVulnSync.js";
|
||||
import { hetznerProblems, hetznerSyncStatuses, imagesOf as hetznerImagesOf, isHetznerFresh, serversOf as hetznerServersOf, type HetznerProblem } from "./hetznerSync.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
|
||||
@@ -119,6 +120,8 @@ export interface HealthSnapshot {
|
||||
dockerMemory?: ContainerMemory;
|
||||
/** Critical vulnerabilities in the images of each Dockhand's Docker hosts, from its last scans, only while that reading is recent (services/dockerVulnSync.ts). */
|
||||
dockerVulns?: { integrationId: number; problems: DockerVulnProblem[] }[];
|
||||
/** What is wrong with each Hetzner Cloud project (a server that went off, a traffic quota nearly used, rescue mode left on, a missing backup), only while that reading is recent (services/hetznerSync.ts). */
|
||||
hetzner?: { integrationId: number; problems: HetznerProblem[] }[];
|
||||
/** Containers to leave alone. */
|
||||
dockerIgnore?: string[];
|
||||
/** The clock the container judgements are made on (the pass's, so a test can set it). */
|
||||
@@ -257,6 +260,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
for (const c of evaluateContainers(snapshot.docker, snapshot.dockerMemory ?? {}, snapshot.dockerNow ?? now, snapshot.dockerIgnore ?? []).conditions) out.push(c);
|
||||
}
|
||||
|
||||
// Hetzner Cloud: servers that went off, traffic quotas, rescue mode left on, backups that are late.
|
||||
for (const h of snapshot.hetzner ?? []) for (const p of h.problems) out.push({ key: p.key, source: `hetzner:${h.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// Critical vulnerabilities in the images Dockhand has scanned.
|
||||
for (const v of snapshot.dockerVulns ?? []) for (const p of v.problems) out.push({ key: p.key, source: `dockhand:${v.integrationId}:vuln`, message: p.message, severity: p.severity });
|
||||
|
||||
@@ -500,6 +506,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
console.error("[health] couldn't judge Nextcloud:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Hetzner Cloud: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsH = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "hetzner"), eq(integrations.enabled, true)));
|
||||
if (rowsH.length > 0) {
|
||||
const statuses = await hetznerSyncStatuses();
|
||||
for (const r of rowsH) {
|
||||
const [state] = await db.select().from(hetznerState).where(eq(hetznerState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isHetznerFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`hetzner:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.hetzner ??= []).push({ integrationId: r.id, problems: hetznerProblems(r.id, r.name, hetznerServersOf(state), hetznerImagesOf(state), Date.now(), checks.backupStaleHours) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Hetzner:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// NetBox: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsL = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "netbox"), eq(integrations.enabled, true)));
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
/**
|
||||
* Keeps what Hetzner Cloud says up to date: every five minutes each Hetzner integration (one project) is read — its servers, volumes, snapshots and backups, primary IPs and Hetzner's
|
||||
* price list — and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Hetzner (whose API allows 3600 requests an hour per project).
|
||||
*
|
||||
* What is raised: a server that **was running and is off** (a server that has been off for a week is left alone, like a stopped container — and one the app never saw running is not
|
||||
* news), a server stuck in a status that should pass (migrating, rebuilding…), a server that has used most of its **traffic** for the month (overage is billed), **rescue mode** left on
|
||||
* (the next reboot would come up in the rescue system), and a server with **backups** enabled whose newest backup is too old. Snapshots, volumes and the cost estimate are shown, not alerted.
|
||||
* Hetzner says nothing of when a server went off, so "since when" is what this app saw: the time of the first read that showed it.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { hetznerState, integrations } from "../db/schema.js";
|
||||
import { createHetznerAdapter, type HetznerImage, type HetznerRead, type HetznerServer, type HetznerVolume } from "../integrations/hetzner/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
const STATUS_FLAG = "hetznerSync";
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
/** A server off for longer than this, that the app saw running before, is no longer news. */
|
||||
export const OFF_NEWS_MS = 7 * 24 * HOUR;
|
||||
/** A status that should pass (starting, migrating, rebuilding…) is a problem after this long. */
|
||||
export const STUCK_AFTER_MS = 30 * MIN;
|
||||
/** Rescue mode switched on and not used within this long: the next reboot would come up in it. */
|
||||
export const RESCUE_AFTER_MS = HOUR;
|
||||
export const TRAFFIC_WARN = 0.8;
|
||||
|
||||
export interface HetznerSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, HetznerSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, HetznerSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function hetznerSyncStatuses(): Promise<Record<number, HetznerSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isHetznerFresh = (status: HetznerSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof hetznerState.$inferSelect;
|
||||
|
||||
/** A server as stored: as read, plus what the app has seen of it between reads. */
|
||||
export interface StoredServer extends HetznerServer {
|
||||
/** The last time a read showed it running; null when none did. */
|
||||
lastRunningAt: string | null;
|
||||
/** When the first read that showed its present status was made. */
|
||||
statusSince: string;
|
||||
/** When the first read that showed rescue mode on was made; null when it is not on. */
|
||||
rescueSince: string | null;
|
||||
}
|
||||
|
||||
export interface HetznerCost {
|
||||
currency: string;
|
||||
servers: number;
|
||||
backups: number;
|
||||
volumes: number;
|
||||
snapshots: number;
|
||||
ips: number;
|
||||
total: number;
|
||||
/** Primary IPs that belong to no server (billed all the same). */
|
||||
unassignedIps: number;
|
||||
/** Servers whose price could not be read, so the total is short by them. */
|
||||
unpriced: number;
|
||||
}
|
||||
|
||||
const parseList = <T>(stored: string | null): T[] | null => {
|
||||
if (stored === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? (v as T[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
export const serversOf = (row: Pick<StateRow, "servers">): StoredServer[] => parseList<StoredServer>(row.servers) ?? [];
|
||||
export const volumesOf = (row: Pick<StateRow, "volumes">): HetznerVolume[] | null => parseList<HetznerVolume>(row.volumes);
|
||||
export const imagesOf = (row: Pick<StateRow, "images">): HetznerImage[] | null => parseList<HetznerImage>(row.images);
|
||||
export const costOf = (row: Pick<StateRow, "cost">): HetznerCost | null => {
|
||||
if (!row.cost) return null;
|
||||
try {
|
||||
return JSON.parse(row.cost) as HetznerCost;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/** Pure: what the app has seen between reads, carried from the last read of the same server. A server first seen starts its clocks now. */
|
||||
export function carryTimes(servers: HetznerServer[], previous: StoredServer[], now: Date): StoredServer[] {
|
||||
const before = new Map(previous.map((p) => [p.id, p]));
|
||||
const at = now.toISOString();
|
||||
return servers.map((s) => {
|
||||
const p = before.get(s.id);
|
||||
return {
|
||||
...s,
|
||||
lastRunningAt: s.status === "running" ? at : (p?.lastRunningAt ?? null),
|
||||
statusSince: p && p.status === s.status ? p.statusSince : at,
|
||||
rescueSince: s.rescueEnabled ? (p?.rescueEnabled ? (p.rescueSince ?? at) : at) : null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Pure: roughly what the project costs per month at Hetzner's list prices, including VAT; null when the price list could not be read. Load balancers, floating IPs and traffic over the quota are not in it. */
|
||||
export function estimateCost(read: Pick<HetznerRead, "servers" | "volumes" | "images" | "primaryIps" | "prices">): HetznerCost | null {
|
||||
const p = read.prices;
|
||||
if (!p) return null;
|
||||
let servers = 0;
|
||||
let backups = 0;
|
||||
let unpriced = 0;
|
||||
for (const s of read.servers) {
|
||||
if (s.monthlyPrice === null) {
|
||||
unpriced++;
|
||||
continue;
|
||||
}
|
||||
servers += s.monthlyPrice;
|
||||
if (s.backupWindow !== null && p.backupPercent !== null) backups += (s.monthlyPrice * p.backupPercent) / 100;
|
||||
}
|
||||
const volumes = (read.volumes ?? []).reduce((sum, v) => sum + v.size * (p.volumePerGb ?? 0), 0);
|
||||
const snapshots = (read.images ?? []).filter((i) => i.type === "snapshot").reduce((sum, i) => sum + (i.sizeGb ?? 0) * (p.imagePerGb ?? 0), 0);
|
||||
let ips = 0;
|
||||
let unassignedIps = 0;
|
||||
for (const ip of read.primaryIps ?? []) {
|
||||
ips += p.primaryIps.find((x) => x.type === ip.type && x.location === ip.location)?.monthly ?? 0;
|
||||
if (!ip.assigned) unassignedIps++;
|
||||
}
|
||||
const round = (n: number) => Math.round(n * 100) / 100;
|
||||
const parts = { servers: round(servers), backups: round(backups), volumes: round(volumes), snapshots: round(snapshots), ips: round(ips) };
|
||||
return { currency: p.currency, ...parts, total: round(servers + backups + volumes + snapshots + ips), unassignedIps, unpriced };
|
||||
}
|
||||
|
||||
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
|
||||
export async function storeRead(integrationId: number, r: HetznerRead, now: Date = new Date()): Promise<void> {
|
||||
const [previous] = await db.select({ servers: hetznerState.servers }).from(hetznerState).where(eq(hetznerState.integrationId, integrationId)).limit(1);
|
||||
const values = {
|
||||
servers: JSON.stringify(carryTimes(r.servers, previous ? serversOf(previous) : [], now)),
|
||||
volumes: r.volumes && JSON.stringify(r.volumes),
|
||||
images: r.images && JSON.stringify(r.images),
|
||||
cost: ((c) => c && JSON.stringify(c))(estimateCost(r)),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(hetznerState).values({ integrationId, ...values }).onConflictDoUpdate({ target: hetznerState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one project and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncHetzner(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<HetznerRead>): Promise<HetznerSyncStatus> {
|
||||
let status: HetznerSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "hetzner") throw new Error("not a Hetzner integration");
|
||||
const result = await (read ? read(loaded.config) : createHetznerAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[hetzner] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[hetzner] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllHetzner(now: Date = new Date()): Promise<Record<number, HetznerSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "hetzner"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, HetznerSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncHetzner(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetHetzner(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface HetznerProblem {
|
||||
key: string;
|
||||
/** The Hetzner server it is about. */
|
||||
serverId: number;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
/** In the units the pages show (binary, written GB or TB), so an alert and the tile beside it agree. */
|
||||
export const sizeText = (bytes: number): string => {
|
||||
const gb = bytes / 1024 ** 3;
|
||||
return gb >= 1024 ? `${(gb / 1024).toFixed(1)} TB` : `${gb.toFixed(gb >= 100 ? 0 : 1)} GB`;
|
||||
};
|
||||
|
||||
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
|
||||
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / (24 * HOUR)), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
|
||||
|
||||
/**
|
||||
* Pure: what is wrong with one project, from what is stored for it. `backupStaleHours` is the same limit as for Proxmox guests' backups (0 = not judged).
|
||||
*/
|
||||
export function hetznerProblems(id: number, name: string, servers: StoredServer[], images: HetznerImage[] | null, nowMs: number, backupStaleHours: number): HetznerProblem[] {
|
||||
const out: HetznerProblem[] = [];
|
||||
for (const s of [...servers].sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
const since = nowMs - Date.parse(s.statusSince);
|
||||
if (s.status === "off" && s.lastRunningAt !== null && nowMs - Date.parse(s.lastRunningAt) < OFF_NEWS_MS) {
|
||||
out.push({ key: `hetzner-off:${id}:${s.id}`, serverId: s.id, severity: "warning", message: `${name}: server ${s.name} is off — it was running ${ago(nowMs - Date.parse(s.lastRunningAt))} ago (Hetzner bills a server that is off)` });
|
||||
} else if (s.status !== "running" && s.status !== "off" && since >= STUCK_AFTER_MS) {
|
||||
out.push({ key: `hetzner-stuck:${id}:${s.id}`, serverId: s.id, severity: "warning", message: `${name}: server ${s.name} has been "${s.status}" for ${ago(since)}` });
|
||||
}
|
||||
if (s.rescueEnabled && s.rescueSince !== null && nowMs - Date.parse(s.rescueSince) >= RESCUE_AFTER_MS) {
|
||||
out.push({ key: `hetzner-rescue:${id}:${s.id}`, serverId: s.id, severity: "warning", message: `${name}: rescue mode is switched on for ${s.name} (${ago(nowMs - Date.parse(s.rescueSince))}) — its next reboot would come up in the rescue system, not the server` });
|
||||
}
|
||||
if (s.includedTraffic !== null && s.includedTraffic > 0 && s.outgoingTraffic !== null) {
|
||||
const used = s.outgoingTraffic / s.includedTraffic;
|
||||
if (used >= TRAFFIC_WARN) {
|
||||
out.push({
|
||||
key: `hetzner-traffic:${id}:${s.id}`,
|
||||
serverId: s.id,
|
||||
severity: used >= 1 ? "critical" : "warning",
|
||||
message: `${name}: ${s.name} has sent ${sizeText(s.outgoingTraffic)} of its ${sizeText(s.includedTraffic)} traffic this month (${Math.floor(used * 100)}%) — traffic over the quota is billed`,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (backupStaleHours > 0 && s.backupWindow !== null && images !== null && s.status !== "deleting") {
|
||||
const newest = images.filter((i) => i.type === "backup" && i.serverId === s.id && i.created).reduce<number | null>((m, i) => Math.max(m ?? 0, Date.parse(i.created as string)), null);
|
||||
const age = newest === null ? null : nowMs - newest;
|
||||
const limit = backupStaleHours * HOUR;
|
||||
const young = nowMs - Date.parse(s.created ?? "") < limit; // a server (or its backups) that is new has had no time to make one
|
||||
if (age === null ? !young : age > limit) {
|
||||
out.push({ key: `hetzner-backup:${id}:${s.id}`, serverId: s.id, severity: "warning", message: `${name}: backups are enabled for ${s.name} but ${age === null ? "none has been made" : `the newest is ${ago(age)} old`}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── Linking to the servers the app knows ───────────────────────────────────
|
||||
|
||||
/** Pure: which of the app's servers a Hetzner server is. An address wins over a name (names are chosen by hand and may differ); null when none matches. */
|
||||
export function matchAppServer(hz: Pick<HetznerServer, "name" | "ipv4" | "privateIps">, appServers: { id: number; name: string; ips: string[] }[]): { id: number; name: string } | null {
|
||||
const mine = [hz.ipv4, ...hz.privateIps].filter((i): i is string => !!i).map((i) => i.trim().toLowerCase());
|
||||
if (mine.length > 0) {
|
||||
const byIp = appServers.find((a) => a.ips.some((ip) => mine.includes(ip.trim().toLowerCase())));
|
||||
if (byIp) return { id: byIp.id, name: byIp.name };
|
||||
}
|
||||
const byName = appServers.find((a) => a.name.trim().toLowerCase() === hz.name.trim().toLowerCase());
|
||||
return byName ? { id: byName.id, name: byName.name } : null;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initHetznerSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllHetzner().catch((err) => console.error("[hetzner] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 29_000);
|
||||
console.log(`[hetzner] reading Hetzner Cloud every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -37,6 +37,8 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^hetzner-(?:off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^n8n-(?:failed):(\d+)/.exec(key);
|
||||
|
||||
@@ -5,13 +5,14 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
|
||||
import { isNextcloudFresh, nextcloudSyncStatuses } from "./nextcloudSync.js";
|
||||
import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
|
||||
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf } from "./hetznerSync.js";
|
||||
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
|
||||
import { isN8nFresh, n8nSyncStatuses, workflowsOf as n8nWorkflowsOf } from "./n8nSync.js";
|
||||
import { componentsOf as paperlessComponentsOf, failedOf as paperlessFailedOf, isPaperlessFresh, paperlessSyncStatuses } from "./paperlessSync.js";
|
||||
@@ -84,6 +85,24 @@ async function addNextcloudMetrics(out: Exposition, now: Date) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Hetzner Cloud server, for /metrics: whether it is running, its traffic this month against the quota, and the project's monthly cost estimate. Only while the reading is recent. */
|
||||
async function addHetznerMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await hetznerSyncStatuses();
|
||||
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "hetzner"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(hetznerState)) {
|
||||
const row = names.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isHetznerFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
for (const server of hetznerServersOf(s)) {
|
||||
const labels = { project: row.name, server: server.name };
|
||||
out.add("homelab_hetzner_server_running", "gauge", "1 if the Hetzner Cloud server is running, else 0.", server.status === "running" ? 1 : 0, labels);
|
||||
if (server.outgoingTraffic !== null) out.add("homelab_hetzner_traffic_outgoing_bytes", "gauge", "Outgoing traffic of a Hetzner Cloud server this month, in bytes.", server.outgoingTraffic, labels);
|
||||
if (server.includedTraffic !== null) out.add("homelab_hetzner_traffic_included_bytes", "gauge", "The outgoing traffic a Hetzner Cloud server includes per month, in bytes.", server.includedTraffic, labels);
|
||||
}
|
||||
const cost = hetznerCostOf(s);
|
||||
if (cost) out.add("homelab_hetzner_monthly_cost", "gauge", "Estimated monthly cost of a Hetzner Cloud project at list prices, including VAT, in the project's currency.", cost.total, { project: row.name, currency: cost.currency });
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Docker host's vulnerability counts from Dockhand's scans, for /metrics. Only while the reading is recent, and only for hosts whose summary could be read. */
|
||||
async function addDockerVulnMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await dockerVulnSyncStatuses();
|
||||
@@ -406,6 +425,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addNetBoxMetrics(out, now);
|
||||
await addNextcloudMetrics(out, now);
|
||||
await addDockerVulnMetrics(out, now);
|
||||
await addHetznerMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
/**
|
||||
* Hetzner Cloud: reading what its API says (paged, read-only), what is stored and carried between reads, the cost estimate, what is wrong, how a Hetzner server is matched to one the app
|
||||
* knows, the health pass, the alerts, the page's routes and /metrics.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-hetzner-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, integrations, servers, hetznerState } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const adapter = await import("../src/integrations/hetzner/adapter.js");
|
||||
const hz = await import("../src/services/hetznerSync.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { hetznerRouter } = await import("../src/routes/hetzner.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
const TB = 1024 ** 4;
|
||||
const iso = (ms: number) => new Date(ms).toISOString();
|
||||
|
||||
type Read = import("../src/integrations/hetzner/adapter.js").HetznerRead;
|
||||
type Server_ = import("../src/integrations/hetzner/adapter.js").HetznerServer;
|
||||
const server = (extra: Partial<Server_> = {}): Server_ => ({ id: 1, name: "web", status: "running", created: iso(Date.now() - 30 * 24 * HOUR), type: "cx22", cores: 2, memoryGb: 4, diskGb: 40, location: "fsn1", city: "Falkenstein", os: "Ubuntu 24.04", ipv4: "203.0.113.5", ipv6: "2001:db8::/64", privateIps: ["10.0.0.2"], locked: false, rescueEnabled: false, backupWindow: null, includedTraffic: 20 * TB, outgoingTraffic: 1 * TB, ingoingTraffic: 2 * TB, monthlyPrice: 4, ...extra });
|
||||
const PRICES = { currency: "EUR", imagePerGb: 0.012, volumePerGb: 0.05, backupPercent: 20, primaryIps: [{ type: "ipv4", location: "fsn1", monthly: 0.5 }] };
|
||||
const read = (extra: Partial<Read> = {}): Read => ({ servers: [server()], volumes: [], images: [], primaryIps: [], prices: PRICES, ...extra });
|
||||
|
||||
// What Hetzner sends (the OpenAPI file's shapes; prices are strings).
|
||||
const apiServer = (id: number, name: string, extra: Record<string, unknown> = {}) => ({
|
||||
id,
|
||||
name,
|
||||
status: "running",
|
||||
created: "2026-01-02T03:04:05+00:00",
|
||||
public_net: { ipv4: { id: 1, ip: `203.0.113.${id}`, blocked: false, dns_ptr: "x" }, ipv6: { id: 2, ip: "2001:db8::/64", blocked: false, dns_ptr: [] }, floating_ips: [], firewalls: [] },
|
||||
private_net: [{ network: 7, ip: `10.0.0.${id}`, alias_ips: [], mac_address: "86:00:00:aa:bb:cc" }],
|
||||
server_type: { id: 22, name: "cx22", cores: 2, memory: 4, disk: 40, prices: [{ location: "nbg1", price_monthly: { net: "3.0000", gross: "3.5700" } }, { location: "fsn1", price_monthly: { net: "3.2900", gross: "3.9151" } }] },
|
||||
location: { id: 1, name: "fsn1", city: "Falkenstein", country: "DE" },
|
||||
image: { id: 5, type: "system", name: "ubuntu-24.04", description: "Ubuntu 24.04" },
|
||||
rescue_enabled: false,
|
||||
locked: false,
|
||||
backup_window: null,
|
||||
outgoing_traffic: 1_000_000_000,
|
||||
ingoing_traffic: 2_000_000_000,
|
||||
included_traffic: 21_990_232_555_520,
|
||||
...extra,
|
||||
});
|
||||
|
||||
describe("what Hetzner's API says", () => {
|
||||
test("servers: status, type, where, the addresses, the traffic, and the price of its type at its location (including VAT)", () => {
|
||||
const [s] = adapter.parseServers({ servers: [apiServer(5, "web", { backup_window: "22-02", rescue_enabled: true })] });
|
||||
assert.deepEqual([s.id, s.name, s.status, s.type, s.location, s.city, s.os], [5, "web", "running", "cx22", "fsn1", "Falkenstein", "Ubuntu 24.04"]);
|
||||
assert.deepEqual([s.ipv4, s.privateIps, s.rescueEnabled, s.backupWindow, s.includedTraffic, s.outgoingTraffic], ["203.0.113.5", ["10.0.0.5"], true, "22-02", 21_990_232_555_520, 1_000_000_000]);
|
||||
assert.equal(s.monthlyPrice, 3.9151, "the fsn1 price, not the nbg1 one");
|
||||
});
|
||||
|
||||
test("a server without an id or a name is skipped, and missing parts are null, not guessed", () => {
|
||||
const list = adapter.parseServers({ servers: [{ name: "x" }, { id: 2 }, { id: 3, name: "bare" }, "junk"] });
|
||||
assert.deepEqual(list.map((s) => s.name), ["bare"]);
|
||||
assert.deepEqual([list[0].status, list[0].ipv4, list[0].includedTraffic, list[0].monthlyPrice, list[0].backupWindow, list[0].privateIps], ["unknown", null, null, null, null, []]);
|
||||
assert.deepEqual(adapter.parseServers(null), []);
|
||||
});
|
||||
|
||||
test("volumes: the size, and the server it is attached to (none is none)", () => {
|
||||
const v = adapter.parseVolumes({ volumes: [{ id: 1, name: "data", size: 100, server: 5, location: { name: "fsn1" }, status: "available" }, { id: 2, name: "old", size: 50, server: null }, { id: 3 }] });
|
||||
assert.deepEqual(v.map((x) => [x.name, x.size, x.serverId, x.location]), [["data", 100, 5, "fsn1"], ["old", 50, null, null]]);
|
||||
});
|
||||
|
||||
test("snapshots and backups: a backup belongs to bound_to, a snapshot says created_from; system images are not kept", () => {
|
||||
const list = adapter.parseImages({
|
||||
images: [
|
||||
{ id: 1, type: "backup", description: "auto", created: "2026-05-01T00:00:00+00:00", image_size: 3.5, bound_to: 5 },
|
||||
{ id: 2, type: "snapshot", description: "before upgrade", created: "2026-04-01T00:00:00+00:00", image_size: 8, created_from: { id: 5, name: "web" } },
|
||||
{ id: 3, type: "system", description: "Ubuntu" },
|
||||
],
|
||||
});
|
||||
assert.deepEqual(list.map((i) => [i.type, i.serverId, i.sizeGb]), [["backup", 5, 3.5], ["snapshot", 5, 8]]);
|
||||
});
|
||||
|
||||
test("the price list: the per-GB prices, the backup percentage and the primary IP prices; something that is not a price list is not read", () => {
|
||||
const p = adapter.parsePrices({ pricing: { currency: "EUR", image: { price_per_gb_month: { gross: "0.0119" } }, volume: { price_per_gb_month: { gross: "0.0476" } }, server_backup: { percentage: "20" }, primary_ips: [{ type: "ipv4", prices: [{ location: "fsn1", price_monthly: { gross: "0.5950" } }] }] } })!;
|
||||
assert.deepEqual([p.currency, p.imagePerGb, p.volumePerGb, p.backupPercent, p.primaryIps], ["EUR", 0.0119, 0.0476, 20, [{ type: "ipv4", location: "fsn1", monthly: 0.595 }]]);
|
||||
assert.equal(adapter.parsePrices({ message: "no" }), null);
|
||||
});
|
||||
|
||||
test("primary IPs: whether one belongs to a server", () => {
|
||||
const ips = adapter.parsePrimaryIps({ primary_ips: [{ id: 1, ip: "203.0.113.5", type: "ipv4", location: { name: "fsn1" }, assignee_id: 5 }, { id: 2, ip: "203.0.113.6", type: "ipv4", assignee_id: null }, { id: 3, ip: "x", type: "weird" }] });
|
||||
assert.deepEqual(ips.map((i) => [i.ip, i.assigned]), [["203.0.113.5", true], ["203.0.113.6", false]]);
|
||||
});
|
||||
|
||||
test("the next page is the one the meta says; the last has none", () => {
|
||||
assert.equal(adapter.nextPage({ meta: { pagination: { page: 1, next_page: 2 } } }), 2);
|
||||
assert.equal(adapter.nextPage({ meta: { pagination: { page: 2, next_page: null } } }), null);
|
||||
assert.equal(adapter.nextPage({}), null);
|
||||
});
|
||||
|
||||
describe("the adapter against a Hetzner that answers", () => {
|
||||
let fake: Server;
|
||||
let url = "";
|
||||
let tokenOk = true;
|
||||
let volumesStatus = 200;
|
||||
let rateLimited = false;
|
||||
const seen: { method: string | undefined; path: string; auth: string | undefined }[] = [];
|
||||
before(async () => {
|
||||
fake = createHttpServer((req, res) => {
|
||||
const u = new URL(req.url ?? "", "http://x");
|
||||
seen.push({ method: req.method, path: u.pathname + u.search, auth: req.headers.authorization });
|
||||
const send = (code: number, body: unknown) => {
|
||||
res.writeHead(code, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify(body));
|
||||
};
|
||||
if (!tokenOk) return send(401, { error: { code: "unauthorized" } });
|
||||
if (rateLimited) return send(429, { error: { code: "rate_limit_exceeded" } });
|
||||
const page = Number(u.searchParams.get("page") ?? 1);
|
||||
if (u.pathname === "/v1/servers") return send(200, { servers: [apiServer(page, `s${page}`)], meta: { pagination: { page, next_page: page < 2 ? page + 1 : null } } });
|
||||
if (u.pathname === "/v1/volumes") return volumesStatus === 200 ? send(200, { volumes: [{ id: 1, name: "data", size: 10, server: 1, status: "available" }], meta: { pagination: { page: 1, next_page: null } } }) : send(volumesStatus, {});
|
||||
if (u.pathname === "/v1/images") return send(200, { images: u.searchParams.get("type") === "backup" ? [{ id: 9, type: "backup", created: "2026-05-01T00:00:00+00:00", image_size: 2, bound_to: 1 }] : [{ id: 8, type: "snapshot", created: "2026-04-01T00:00:00+00:00", image_size: 5, created_from: { id: 1 } }], meta: { pagination: { page: 1, next_page: null } } });
|
||||
if (u.pathname === "/v1/primary_ips") return send(200, { primary_ips: [], meta: { pagination: { page: 1, next_page: null } } });
|
||||
if (u.pathname === "/v1/pricing") return send(200, { pricing: { currency: "EUR", image: { price_per_gb_month: { gross: "0.0119" } }, volume: { price_per_gb_month: { gross: "0.0476" } }, server_backup: { percentage: "20" }, primary_ips: [] } });
|
||||
send(404, {});
|
||||
});
|
||||
await new Promise<void>((resolve) => fake.listen(0, "127.0.0.1", resolve));
|
||||
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}/v1`;
|
||||
});
|
||||
after(() => fake.close());
|
||||
|
||||
test("every page of the servers is read, with the token as a Bearer, and only ever with GET", async () => {
|
||||
seen.length = 0;
|
||||
const r = await adapter.createHetznerAdapter({ token: "tok", apiUrl: url }).read();
|
||||
assert.deepEqual(r.servers.map((s) => s.name), ["s1", "s2"]);
|
||||
assert.deepEqual([r.volumes?.length, r.images?.map((i) => i.type).sort(), r.prices?.currency], [1, ["backup", "snapshot"], "EUR"]);
|
||||
assert.ok(seen.every((s) => s.method === "GET" && s.auth === "Bearer tok"));
|
||||
assert.ok(seen.some((s) => s.path.startsWith("/v1/servers?per_page=50&page=2")));
|
||||
});
|
||||
|
||||
test("a list that cannot be asked is unknown (null), not empty; the servers are what must work", async () => {
|
||||
volumesStatus = 500;
|
||||
const r = await adapter.createHetznerAdapter({ token: "tok", apiUrl: url }).read();
|
||||
assert.deepEqual([r.volumes, r.servers.length], [null, 2]);
|
||||
assert.equal(hz.estimateCost(r)?.volumes, 0);
|
||||
volumesStatus = 200;
|
||||
});
|
||||
|
||||
test("a token that is refused, and a project over its hourly limit, are said", async () => {
|
||||
tokenOk = false;
|
||||
const ping = await adapter.createHetznerAdapter({ token: "bad", apiUrl: url }).ping();
|
||||
assert.equal(ping.ok, false);
|
||||
assert.match(ping.error ?? "", /did not accept the token/);
|
||||
tokenOk = true;
|
||||
rateLimited = true;
|
||||
await assert.rejects(() => adapter.createHetznerAdapter({ token: "t", apiUrl: url }).read(), /too many requests/);
|
||||
rateLimited = false;
|
||||
assert.equal((await adapter.createHetznerAdapter({ token: "t", apiUrl: url }).ping()).ok, true);
|
||||
assert.equal((await adapter.createHetznerAdapter({ token: "t", apiUrl: "http://127.0.0.1:1/v1" }).ping()).ok, false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is carried between reads", () => {
|
||||
const t0 = new Date("2026-06-01T10:00:00Z");
|
||||
test("a server first seen starts its clocks now; one seen running remembers when", () => {
|
||||
const a = hz.carryTimes([server({ status: "running" }), server({ id: 2, name: "off", status: "off" })], [], t0);
|
||||
assert.deepEqual([a[0].lastRunningAt, a[0].statusSince, a[1].lastRunningAt, a[1].statusSince], [t0.toISOString(), t0.toISOString(), null, t0.toISOString()]);
|
||||
});
|
||||
|
||||
test("when it goes off the last time it ran is kept, and the status clock starts at the first read that showed it", () => {
|
||||
const t1 = new Date(t0.getTime() + 5 * MIN);
|
||||
const t2 = new Date(t0.getTime() + 10 * MIN);
|
||||
const running = hz.carryTimes([server()], [], t0);
|
||||
const off1 = hz.carryTimes([server({ status: "off" })], running, t1);
|
||||
const off2 = hz.carryTimes([server({ status: "off" })], off1, t2);
|
||||
assert.deepEqual([off1[0].lastRunningAt, off1[0].statusSince], [t0.toISOString(), t1.toISOString()]);
|
||||
assert.deepEqual([off2[0].lastRunningAt, off2[0].statusSince], [t0.toISOString(), t1.toISOString()], "the clock does not restart while the status is the same");
|
||||
});
|
||||
|
||||
test("rescue mode's clock starts when it is first seen on, and stops when it is off", () => {
|
||||
const t1 = new Date(t0.getTime() + 5 * MIN);
|
||||
const on = hz.carryTimes([server({ rescueEnabled: true })], hz.carryTimes([server()], [], t0), t0);
|
||||
const still = hz.carryTimes([server({ rescueEnabled: true })], on, t1);
|
||||
assert.equal(still[0].rescueSince, t0.toISOString());
|
||||
assert.equal(hz.carryTimes([server()], still, t1)[0].rescueSince, null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the cost estimate", () => {
|
||||
test("servers, their backups (a percentage of the server), volumes, snapshots and primary IPs, at list prices", () => {
|
||||
const c = hz.estimateCost({
|
||||
servers: [server({ monthlyPrice: 10, backupWindow: "22-02" }), server({ id: 2, name: "b", monthlyPrice: 5 })],
|
||||
volumes: [{ id: 1, name: "v", size: 100, serverId: 1, location: "fsn1", status: "available" }],
|
||||
images: [{ id: 1, type: "snapshot", description: null, created: null, sizeGb: 10, serverId: 1 }, { id: 2, type: "backup", description: null, created: null, sizeGb: 30, serverId: 1 }],
|
||||
primaryIps: [{ id: 1, ip: "1.1.1.1", type: "ipv4", location: "fsn1", assigned: true }, { id: 2, ip: "1.1.1.2", type: "ipv4", location: "fsn1", assigned: false }],
|
||||
prices: PRICES,
|
||||
})!;
|
||||
assert.deepEqual({ ...c }, { currency: "EUR", servers: 15, backups: 2, volumes: 5, snapshots: 0.12, ips: 1, total: 23.12, unassignedIps: 1, unpriced: 0 });
|
||||
});
|
||||
|
||||
test("without the price list there is no estimate; a server whose price is not known is counted as unpriced", () => {
|
||||
assert.equal(hz.estimateCost(read({ prices: null })), null);
|
||||
assert.equal(hz.estimateCost(read({ servers: [server({ monthlyPrice: null })] }))?.unpriced, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is wrong", () => {
|
||||
const NOW = Date.parse("2026-06-10T12:00:00Z");
|
||||
const stored = (extra: Partial<Server_> = {}, times: Partial<import("../src/services/hetznerSync.js").StoredServer> = {}) => ({ ...server({ created: iso(NOW - 30 * 24 * HOUR), ...extra }), lastRunningAt: iso(NOW - HOUR), statusSince: iso(NOW - HOUR), rescueSince: null, ...times });
|
||||
const keys = (list: ReturnType<typeof hz.hetznerProblems>) => list.map((p) => `${p.key}=${p.severity}`);
|
||||
const problems = (s: ReturnType<typeof stored>, images: Read["images"] = [], hours = 48) => hz.hetznerProblems(1, "Prod", [s], images, NOW, hours);
|
||||
|
||||
test("a server that is running and well says nothing", () => {
|
||||
assert.deepEqual(problems(stored()), []);
|
||||
});
|
||||
|
||||
test("a server that was running and is off is a warning; one off for a week, or never seen running, is left alone", () => {
|
||||
assert.deepEqual(keys(problems(stored({ status: "off" }))), ["hetzner-off:1:1=warning"]);
|
||||
assert.match(problems(stored({ status: "off" }))[0].message, /Prod: server web is off — it was running 1 hour ago \(Hetzner bills a server that is off\)/);
|
||||
assert.deepEqual(problems(stored({ status: "off" }, { lastRunningAt: iso(NOW - 8 * 24 * HOUR) })), []);
|
||||
assert.deepEqual(problems(stored({ status: "off" }, { lastRunningAt: null })), []);
|
||||
});
|
||||
|
||||
test("a status that should pass is a problem after half an hour, and not before", () => {
|
||||
assert.deepEqual(problems(stored({ status: "migrating" }, { statusSince: iso(NOW - 10 * MIN) })), []);
|
||||
const p = problems(stored({ status: "migrating" }, { statusSince: iso(NOW - 45 * MIN) }));
|
||||
assert.deepEqual(keys(p), ["hetzner-stuck:1:1=warning"]);
|
||||
assert.match(p[0].message, /has been "migrating" for 45 minutes/);
|
||||
});
|
||||
|
||||
test("rescue mode left on for an hour is a warning, and says what a reboot would do", () => {
|
||||
assert.deepEqual(problems(stored({ rescueEnabled: true }, { rescueSince: iso(NOW - 30 * MIN) })), []);
|
||||
const p = problems(stored({ rescueEnabled: true }, { rescueSince: iso(NOW - 3 * HOUR) }));
|
||||
assert.deepEqual(keys(p), ["hetzner-rescue:1:1=warning"]);
|
||||
assert.match(p[0].message, /next reboot would come up in the rescue system/);
|
||||
});
|
||||
|
||||
test("traffic: a warning from 80% of the month's quota, critical at 100%, in the units the page shows", () => {
|
||||
assert.deepEqual(problems(stored({ outgoingTraffic: 15 * TB })), []);
|
||||
const warn = problems(stored({ outgoingTraffic: 17 * TB }));
|
||||
assert.deepEqual(keys(warn), ["hetzner-traffic:1:1=warning"]);
|
||||
assert.match(warn[0].message, /web has sent 17\.0 TB of its 20\.0 TB traffic this month \(85%\)/);
|
||||
assert.deepEqual(keys(problems(stored({ outgoingTraffic: 21 * TB }))), ["hetzner-traffic:1:1=critical"]);
|
||||
assert.deepEqual(problems(stored({ outgoingTraffic: null })), []);
|
||||
assert.deepEqual(problems(stored({ includedTraffic: null, outgoingTraffic: 99 * TB })), []);
|
||||
});
|
||||
|
||||
describe("backups", () => {
|
||||
const backup = (ageHours: number, serverId = 1) => ({ id: 1, type: "backup" as const, description: null, created: iso(NOW - ageHours * HOUR), sizeGb: 2, serverId });
|
||||
test("enabled and recent is fine; enabled and old, or none for an old server, is a warning; not enabled is not judged", () => {
|
||||
const on = stored({ backupWindow: "22-02" });
|
||||
assert.deepEqual(problems(on, [backup(20)]), []);
|
||||
assert.match(problems(on, [backup(100)])[0].message, /backups are enabled for web but the newest is 4 days old/);
|
||||
assert.match(problems(on, [])[0].message, /but none has been made/);
|
||||
assert.deepEqual(problems(on, [backup(100, 99)]).length, 1, "another server's backup does not count");
|
||||
assert.deepEqual(problems(stored(), []), []);
|
||||
});
|
||||
test("a new server has had no time to make one; the limit is the setting, and 0 turns it off; an unknown list says nothing", () => {
|
||||
const fresh = stored({ backupWindow: "22-02", created: iso(NOW - 5 * HOUR) });
|
||||
assert.deepEqual(problems(fresh, []), []);
|
||||
assert.deepEqual(problems(stored({ backupWindow: "22-02" }), [backup(100)], 0), []);
|
||||
assert.deepEqual(problems(stored({ backupWindow: "22-02" }), [backup(100)], 200), []);
|
||||
assert.deepEqual(problems(stored({ backupWindow: "22-02" }), null), []);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("which of the app's servers a Hetzner server is", () => {
|
||||
const app = [{ id: 1, name: "web-prod", ips: ["203.0.113.5", "192.168.1.5"] }, { id: 2, name: "Db", ips: [] }, { id: 3, name: "other", ips: ["10.0.0.2"] }];
|
||||
test("by an address first, then by name; a server the app does not know matches none", () => {
|
||||
assert.equal(hz.matchAppServer({ name: "anything", ipv4: "203.0.113.5", privateIps: [] }, app)?.id, 1);
|
||||
assert.equal(hz.matchAppServer({ name: "x", ipv4: null, privateIps: ["10.0.0.2"] }, app)?.id, 3, "a private address counts");
|
||||
assert.equal(hz.matchAppServer({ name: "db", ipv4: "198.51.100.9", privateIps: [] }, app)?.id, 2, "names are compared without regard to case");
|
||||
assert.equal(hz.matchAppServer({ name: "web", ipv4: "198.51.100.9", privateIps: [] }, app), null);
|
||||
});
|
||||
test("an address wins over a name that belongs to another server", () => {
|
||||
assert.equal(hz.matchAppServer({ name: "Db", ipv4: "203.0.113.5", privateIps: [] }, app)?.id, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is stored", () => {
|
||||
let id = 0;
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "hetzner", name: "Project", baseUrl: "https://api.hetzner.cloud", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
});
|
||||
const row = async () => (await db.select().from(hetznerState).where(eq(hetznerState.integrationId, id)))[0];
|
||||
after(async () => {
|
||||
await hz.forgetHetzner(id);
|
||||
await db.delete(integrations).where(eq(integrations.id, id)); // so its servers are not matched by the tests that follow
|
||||
});
|
||||
|
||||
test("the state is replaced at each read, carrying what was seen between them; an unknown list is stored as null", async () => {
|
||||
const t0 = new Date();
|
||||
await hz.storeRead(id, read({ volumes: null, prices: null }), t0);
|
||||
let r = await row();
|
||||
assert.deepEqual([hz.serversOf(r).length, hz.volumesOf(r), hz.costOf(r)], [1, null, null]);
|
||||
await hz.storeRead(id, read({ servers: [server({ status: "off" })], volumes: [{ id: 1, name: "v", size: 10, serverId: 1, location: "fsn1", status: "available" }] }), new Date(t0.getTime() + 5 * MIN));
|
||||
r = await row();
|
||||
const [s] = hz.serversOf(r);
|
||||
assert.deepEqual([s.status, s.lastRunningAt, hz.volumesOf(r)?.length, hz.costOf(r)?.currency], ["off", t0.toISOString(), 1, "EUR"]);
|
||||
});
|
||||
|
||||
test("a read that failed is said, and what was stored is left alone", async () => {
|
||||
const ok = await hz.syncHetzner(id, new Date(), async () => read({ servers: [server({ name: "kept" })] }));
|
||||
assert.equal(ok.ok, true);
|
||||
const failed = await hz.syncHetzner(id, new Date(), async () => {
|
||||
throw new Error("connection refused");
|
||||
});
|
||||
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
|
||||
assert.equal(failed.at, ok.at);
|
||||
assert.equal(hz.serversOf(await row())[0].name, "kept");
|
||||
assert.equal(hz.isHetznerFresh((await hz.hetznerSyncStatuses())[id], Date.now()), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the health pass, the page and /metrics", () => {
|
||||
let hzId = 0;
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const ids: Record<string, number> = {};
|
||||
const fresh = () => setInternalFlag("hetznerSync", JSON.stringify({ [hzId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
const stale = () => setInternalFlag("hetznerSync", JSON.stringify({ [hzId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
hzId = (await db.insert(integrations).values({ type: "hetzner", name: "Prod project", baseUrl: "https://api.hetzner.cloud", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
|
||||
const mk = async (name: string, ips: string[]) => (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, ipAddresses: JSON.stringify(ips) }).returning({ id: servers.id }))[0].id;
|
||||
ids.web = await mk("web-agent", ["203.0.113.5"]);
|
||||
ids.other = await mk("elsewhere", ["192.168.9.9"]);
|
||||
// web: running but nearly out of traffic; db: was running and is now off; blog: rescue left on for hours.
|
||||
const t0 = new Date(Date.now() - 3 * HOUR);
|
||||
await hz.storeRead(hzId, read({ servers: [server({ id: 1, name: "web", outgoingTraffic: 19 * TB, backupWindow: "22-02" }), server({ id: 2, name: "db", ipv4: "203.0.113.6", status: "running" }), server({ id: 3, name: "blog", ipv4: "203.0.113.7", rescueEnabled: true })], volumes: [{ id: 1, name: "data", size: 100, serverId: 1, location: "fsn1", status: "available" }, { id: 2, name: "orphan", size: 20, serverId: null, location: "fsn1", status: "available" }], images: [{ id: 1, type: "snapshot", description: "before upgrade", created: iso(Date.now() - 40 * 24 * HOUR), sizeGb: 8, serverId: 1 }] }), t0);
|
||||
await hz.storeRead(hzId, read({ servers: [server({ id: 1, name: "web", outgoingTraffic: 19 * TB, backupWindow: "22-02" }), server({ id: 2, name: "db", ipv4: "203.0.113.6", status: "off" }), server({ id: 3, name: "blog", ipv4: "203.0.113.7", rescueEnabled: true })], volumes: [{ id: 1, name: "data", size: 100, serverId: 1, location: "fsn1", status: "available" }, { id: 2, name: "orphan", size: 20, serverId: null, location: "fsn1", status: "available" }], images: [{ id: 1, type: "snapshot", description: "before upgrade", created: iso(Date.now() - 40 * 24 * HOUR), sizeGb: 8, serverId: 1 }] }), new Date());
|
||||
await fresh();
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/hetzner", hetznerRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/hetzner`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
|
||||
let { snapshot, held } = await collectSnapshot();
|
||||
assert.deepEqual(snapshot.hetzner?.map((h) => h.integrationId), [hzId]);
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const hetzner = list.filter((x) => x.key.startsWith("hetzner-")).map((x) => `${x.key}=${x.severity}`).sort();
|
||||
assert.deepEqual(hetzner, [`hetzner-backup:${hzId}:1=warning`, `hetzner-off:${hzId}:2=warning`, `hetzner-rescue:${hzId}:3=warning`, `hetzner-traffic:${hzId}:1=warning`].sort());
|
||||
const off = list.find((x) => x.key === `hetzner-off:${hzId}:2`)!;
|
||||
assert.equal(off.source, `hetzner:${hzId}`);
|
||||
for (const k of ["off", "stuck", "rescue", "traffic", "backup"]) assert.equal(subjectOfConditionKey(`hetzner-${k}:${hzId}:2`), `integration:${hzId}`, "maintenance on the project silences its alerts");
|
||||
await stale();
|
||||
({ snapshot, held } = await collectSnapshot());
|
||||
assert.deepEqual([snapshot.hetzner ?? [], held.has(`hetzner:${hzId}`)], [[], true]);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("the Alerts page shows them with where to look: the backup under Backups, the traffic under Monitoring, the rest under Services", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const find = (kind: string, server: number) => report.alerts.find((x) => x.id.endsWith(`hetzner-${kind}:${hzId}:${server}`))!;
|
||||
assert.deepEqual([find("off", 2).category, find("off", 2).source, find("off", 2).link, find("off", 2).target?.type], ["services", "Hetzner", "/hetzner", "integration"]);
|
||||
assert.equal(find("traffic", 1).category, "monitoring");
|
||||
assert.equal(find("backup", 1).category, "backup");
|
||||
assert.equal(find("rescue", 3).category, "services");
|
||||
});
|
||||
|
||||
test("the page: each server with the app's server it is (or none), the volumes with what they hang on, the snapshots, the cost and what is wrong", async () => {
|
||||
const body = (await (await fetch(base)).json()) as { projects: { id: number; fresh: boolean; problems: unknown[]; state: { servers: { name: string; linked: { id: number; name: string } | null }[]; volumes: { name: string; serverName: string | null }[] | null; images: { description: string; serverName: string | null }[]; cost: { currency: string; total: number } } }[] };
|
||||
const p = body.projects.find((x) => x.id === hzId)!;
|
||||
assert.equal(p.fresh, true);
|
||||
assert.deepEqual(p.state.servers.map((s) => [s.name, s.linked?.name ?? null]), [["web", "web-agent"], ["db", null], ["blog", null]]);
|
||||
assert.deepEqual(p.state.volumes?.map((v) => [v.name, v.serverName]), [["data", "web"], ["orphan", null]]);
|
||||
assert.deepEqual(p.state.images.map((i) => [i.description, i.serverName]), [["before upgrade", "web"]]);
|
||||
assert.equal(p.state.cost.currency, "EUR");
|
||||
assert.equal(p.problems.length, 4);
|
||||
await stale();
|
||||
const old = (await (await fetch(base)).json()) as typeof body;
|
||||
assert.deepEqual(old.projects.find((x) => x.id === hzId)!.problems, [], "an old reading says nothing is wrong");
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("the server's own page gets its Hetzner server, with what is wrong with it, and a server that is not one gets none", async () => {
|
||||
const mine = (await (await fetch(`${base}/for-server/${ids.web}`)).json()) as { match: { integrationName: string; fresh: boolean; server: { name: string; includedTraffic: number }; problems: { key: string }[]; snapshots: number; newestBackup: string | null } | null };
|
||||
assert.deepEqual([mine.match?.integrationName, mine.match?.server.name, mine.match?.snapshots, mine.match?.newestBackup], ["Prod project", "web", 1, null]);
|
||||
assert.deepEqual(mine.match?.problems.map((p) => p.key).sort(), [`hetzner-backup:${hzId}:1`, `hetzner-traffic:${hzId}:1`]);
|
||||
const none = (await (await fetch(`${base}/for-server/${ids.other}`)).json()) as { match: unknown };
|
||||
assert.equal(none.match, null);
|
||||
assert.equal((await fetch(`${base}/for-server/999999`)).status, 404);
|
||||
assert.equal((await fetch(`${base}/for-server/abc`)).status, 400);
|
||||
});
|
||||
|
||||
test("/metrics has whether each server runs, its traffic against the quota and the project's cost — only while the reading is recent", async () => {
|
||||
let text = await buildMetrics(new Date());
|
||||
assert.match(text, /homelab_hetzner_server_running\{project="Prod project",server="web"\} 1/);
|
||||
assert.match(text, /homelab_hetzner_server_running\{project="Prod project",server="db"\} 0/);
|
||||
assert.match(text, /homelab_hetzner_traffic_included_bytes\{project="Prod project",server="web"\}/);
|
||||
assert.match(text, /homelab_hetzner_monthly_cost\{[^}]*currency="EUR"[^}]*\} \d/);
|
||||
await stale();
|
||||
text = await buildMetrics(new Date());
|
||||
assert.doesNotMatch(text, /homelab_hetzner/);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("taking the integration away takes what was read with it", async () => {
|
||||
await hz.forgetHetzner(hzId);
|
||||
assert.equal((await hz.hetznerSyncStatuses())[hzId], undefined);
|
||||
await db.delete(integrations).where(eq(integrations.id, hzId));
|
||||
assert.deepEqual(await db.select().from(hetznerState).where(eq(hetznerState.integrationId, hzId)), []);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user