Hetzner Cloud integration: servers, traffic quota, backups, volumes and cost
One integration per project (read-only API token, read every 5 minutes). Alerts for a server that was running and is off, stuck statuses, traffic at 80%/100% of the monthly quota, rescue mode left on, and late backups. The Hetzner page lists servers, volumes, snapshots and a cost estimate; a server the app knows by its agent gets a Hetzner card on its own page. /metrics and docs updated. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5a2b320175
commit
24b708abbe
35 files changed
+9430
-7
No files matched your search
@@ -83,6 +83,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
n8n: { label: "n8n", link: "/n8n" },
|
||||
netbox: { label: "NetBox", link: "/netbox" },
|
||||
nextcloud: { label: "Nextcloud", link: "/nextcloud" },
|
||||
hetzner: { label: "Hetzner", link: "/hetzner" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -121,6 +122,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^docker-vuln:(\d+):/.exec(key);
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^hetzner-(off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
|
||||
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
||||
@@ -261,7 +264,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
@@ -271,7 +274,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities } from "../db/schema.js";
|
||||
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -29,6 +29,7 @@ import { isN8nFresh, n8nProblems, n8nSyncStatuses, type N8nProblem } from "./n8n
|
||||
import { isNetBoxFresh, netboxProblems, netboxSyncStatuses, type NetBoxProblem } from "./netboxSync.js";
|
||||
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type NextcloudProblem } from "./nextcloudSync.js";
|
||||
import { dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, type DockerVulnProblem } from "./dockerVulnSync.js";
|
||||
import { hetznerProblems, hetznerSyncStatuses, imagesOf as hetznerImagesOf, isHetznerFresh, serversOf as hetznerServersOf, type HetznerProblem } from "./hetznerSync.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
|
||||
@@ -119,6 +120,8 @@ export interface HealthSnapshot {
|
||||
dockerMemory?: ContainerMemory;
|
||||
/** Critical vulnerabilities in the images of each Dockhand's Docker hosts, from its last scans, only while that reading is recent (services/dockerVulnSync.ts). */
|
||||
dockerVulns?: { integrationId: number; problems: DockerVulnProblem[] }[];
|
||||
/** What is wrong with each Hetzner Cloud project (a server that went off, a traffic quota nearly used, rescue mode left on, a missing backup), only while that reading is recent (services/hetznerSync.ts). */
|
||||
hetzner?: { integrationId: number; problems: HetznerProblem[] }[];
|
||||
/** Containers to leave alone. */
|
||||
dockerIgnore?: string[];
|
||||
/** The clock the container judgements are made on (the pass's, so a test can set it). */
|
||||
@@ -257,6 +260,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
for (const c of evaluateContainers(snapshot.docker, snapshot.dockerMemory ?? {}, snapshot.dockerNow ?? now, snapshot.dockerIgnore ?? []).conditions) out.push(c);
|
||||
}
|
||||
|
||||
// Hetzner Cloud: servers that went off, traffic quotas, rescue mode left on, backups that are late.
|
||||
for (const h of snapshot.hetzner ?? []) for (const p of h.problems) out.push({ key: p.key, source: `hetzner:${h.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// Critical vulnerabilities in the images Dockhand has scanned.
|
||||
for (const v of snapshot.dockerVulns ?? []) for (const p of v.problems) out.push({ key: p.key, source: `dockhand:${v.integrationId}:vuln`, message: p.message, severity: p.severity });
|
||||
|
||||
@@ -500,6 +506,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
console.error("[health] couldn't judge Nextcloud:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Hetzner Cloud: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsH = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "hetzner"), eq(integrations.enabled, true)));
|
||||
if (rowsH.length > 0) {
|
||||
const statuses = await hetznerSyncStatuses();
|
||||
for (const r of rowsH) {
|
||||
const [state] = await db.select().from(hetznerState).where(eq(hetznerState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isHetznerFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`hetzner:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.hetzner ??= []).push({ integrationId: r.id, problems: hetznerProblems(r.id, r.name, hetznerServersOf(state), hetznerImagesOf(state), Date.now(), checks.backupStaleHours) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Hetzner:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// NetBox: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsL = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "netbox"), eq(integrations.enabled, true)));
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
/**
|
||||
* Keeps what Hetzner Cloud says up to date: every five minutes each Hetzner integration (one project) is read — its servers, volumes, snapshots and backups, primary IPs and Hetzner's
|
||||
* price list — and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Hetzner (whose API allows 3600 requests an hour per project).
|
||||
*
|
||||
* What is raised: a server that **was running and is off** (a server that has been off for a week is left alone, like a stopped container — and one the app never saw running is not
|
||||
* news), a server stuck in a status that should pass (migrating, rebuilding…), a server that has used most of its **traffic** for the month (overage is billed), **rescue mode** left on
|
||||
* (the next reboot would come up in the rescue system), and a server with **backups** enabled whose newest backup is too old. Snapshots, volumes and the cost estimate are shown, not alerted.
|
||||
* Hetzner says nothing of when a server went off, so "since when" is what this app saw: the time of the first read that showed it.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { hetznerState, integrations } from "../db/schema.js";
|
||||
import { createHetznerAdapter, type HetznerImage, type HetznerRead, type HetznerServer, type HetznerVolume } from "../integrations/hetzner/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
const STATUS_FLAG = "hetznerSync";
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
/** A server off for longer than this, that the app saw running before, is no longer news. */
|
||||
export const OFF_NEWS_MS = 7 * 24 * HOUR;
|
||||
/** A status that should pass (starting, migrating, rebuilding…) is a problem after this long. */
|
||||
export const STUCK_AFTER_MS = 30 * MIN;
|
||||
/** Rescue mode switched on and not used within this long: the next reboot would come up in it. */
|
||||
export const RESCUE_AFTER_MS = HOUR;
|
||||
export const TRAFFIC_WARN = 0.8;
|
||||
|
||||
export interface HetznerSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, HetznerSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, HetznerSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function hetznerSyncStatuses(): Promise<Record<number, HetznerSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isHetznerFresh = (status: HetznerSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof hetznerState.$inferSelect;
|
||||
|
||||
/** A server as stored: as read, plus what the app has seen of it between reads. */
|
||||
export interface StoredServer extends HetznerServer {
|
||||
/** The last time a read showed it running; null when none did. */
|
||||
lastRunningAt: string | null;
|
||||
/** When the first read that showed its present status was made. */
|
||||
statusSince: string;
|
||||
/** When the first read that showed rescue mode on was made; null when it is not on. */
|
||||
rescueSince: string | null;
|
||||
}
|
||||
|
||||
export interface HetznerCost {
|
||||
currency: string;
|
||||
servers: number;
|
||||
backups: number;
|
||||
volumes: number;
|
||||
snapshots: number;
|
||||
ips: number;
|
||||
total: number;
|
||||
/** Primary IPs that belong to no server (billed all the same). */
|
||||
unassignedIps: number;
|
||||
/** Servers whose price could not be read, so the total is short by them. */
|
||||
unpriced: number;
|
||||
}
|
||||
|
||||
const parseList = <T>(stored: string | null): T[] | null => {
|
||||
if (stored === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? (v as T[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
export const serversOf = (row: Pick<StateRow, "servers">): StoredServer[] => parseList<StoredServer>(row.servers) ?? [];
|
||||
export const volumesOf = (row: Pick<StateRow, "volumes">): HetznerVolume[] | null => parseList<HetznerVolume>(row.volumes);
|
||||
export const imagesOf = (row: Pick<StateRow, "images">): HetznerImage[] | null => parseList<HetznerImage>(row.images);
|
||||
export const costOf = (row: Pick<StateRow, "cost">): HetznerCost | null => {
|
||||
if (!row.cost) return null;
|
||||
try {
|
||||
return JSON.parse(row.cost) as HetznerCost;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/** Pure: what the app has seen between reads, carried from the last read of the same server. A server first seen starts its clocks now. */
|
||||
export function carryTimes(servers: HetznerServer[], previous: StoredServer[], now: Date): StoredServer[] {
|
||||
const before = new Map(previous.map((p) => [p.id, p]));
|
||||
const at = now.toISOString();
|
||||
return servers.map((s) => {
|
||||
const p = before.get(s.id);
|
||||
return {
|
||||
...s,
|
||||
lastRunningAt: s.status === "running" ? at : (p?.lastRunningAt ?? null),
|
||||
statusSince: p && p.status === s.status ? p.statusSince : at,
|
||||
rescueSince: s.rescueEnabled ? (p?.rescueEnabled ? (p.rescueSince ?? at) : at) : null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Pure: roughly what the project costs per month at Hetzner's list prices, including VAT; null when the price list could not be read. Load balancers, floating IPs and traffic over the quota are not in it. */
|
||||
export function estimateCost(read: Pick<HetznerRead, "servers" | "volumes" | "images" | "primaryIps" | "prices">): HetznerCost | null {
|
||||
const p = read.prices;
|
||||
if (!p) return null;
|
||||
let servers = 0;
|
||||
let backups = 0;
|
||||
let unpriced = 0;
|
||||
for (const s of read.servers) {
|
||||
if (s.monthlyPrice === null) {
|
||||
unpriced++;
|
||||
continue;
|
||||
}
|
||||
servers += s.monthlyPrice;
|
||||
if (s.backupWindow !== null && p.backupPercent !== null) backups += (s.monthlyPrice * p.backupPercent) / 100;
|
||||
}
|
||||
const volumes = (read.volumes ?? []).reduce((sum, v) => sum + v.size * (p.volumePerGb ?? 0), 0);
|
||||
const snapshots = (read.images ?? []).filter((i) => i.type === "snapshot").reduce((sum, i) => sum + (i.sizeGb ?? 0) * (p.imagePerGb ?? 0), 0);
|
||||
let ips = 0;
|
||||
let unassignedIps = 0;
|
||||
for (const ip of read.primaryIps ?? []) {
|
||||
ips += p.primaryIps.find((x) => x.type === ip.type && x.location === ip.location)?.monthly ?? 0;
|
||||
if (!ip.assigned) unassignedIps++;
|
||||
}
|
||||
const round = (n: number) => Math.round(n * 100) / 100;
|
||||
const parts = { servers: round(servers), backups: round(backups), volumes: round(volumes), snapshots: round(snapshots), ips: round(ips) };
|
||||
return { currency: p.currency, ...parts, total: round(servers + backups + volumes + snapshots + ips), unassignedIps, unpriced };
|
||||
}
|
||||
|
||||
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
|
||||
export async function storeRead(integrationId: number, r: HetznerRead, now: Date = new Date()): Promise<void> {
|
||||
const [previous] = await db.select({ servers: hetznerState.servers }).from(hetznerState).where(eq(hetznerState.integrationId, integrationId)).limit(1);
|
||||
const values = {
|
||||
servers: JSON.stringify(carryTimes(r.servers, previous ? serversOf(previous) : [], now)),
|
||||
volumes: r.volumes && JSON.stringify(r.volumes),
|
||||
images: r.images && JSON.stringify(r.images),
|
||||
cost: ((c) => c && JSON.stringify(c))(estimateCost(r)),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(hetznerState).values({ integrationId, ...values }).onConflictDoUpdate({ target: hetznerState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one project and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncHetzner(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<HetznerRead>): Promise<HetznerSyncStatus> {
|
||||
let status: HetznerSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "hetzner") throw new Error("not a Hetzner integration");
|
||||
const result = await (read ? read(loaded.config) : createHetznerAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[hetzner] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[hetzner] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllHetzner(now: Date = new Date()): Promise<Record<number, HetznerSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "hetzner"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, HetznerSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncHetzner(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetHetzner(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface HetznerProblem {
|
||||
key: string;
|
||||
/** The Hetzner server it is about. */
|
||||
serverId: number;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
/** In the units the pages show (binary, written GB or TB), so an alert and the tile beside it agree. */
|
||||
export const sizeText = (bytes: number): string => {
|
||||
const gb = bytes / 1024 ** 3;
|
||||
return gb >= 1024 ? `${(gb / 1024).toFixed(1)} TB` : `${gb.toFixed(gb >= 100 ? 0 : 1)} GB`;
|
||||
};
|
||||
|
||||
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
|
||||
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / (24 * HOUR)), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
|
||||
|
||||
/**
|
||||
* Pure: what is wrong with one project, from what is stored for it. `backupStaleHours` is the same limit as for Proxmox guests' backups (0 = not judged).
|
||||
*/
|
||||
export function hetznerProblems(id: number, name: string, servers: StoredServer[], images: HetznerImage[] | null, nowMs: number, backupStaleHours: number): HetznerProblem[] {
|
||||
const out: HetznerProblem[] = [];
|
||||
for (const s of [...servers].sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
const since = nowMs - Date.parse(s.statusSince);
|
||||
if (s.status === "off" && s.lastRunningAt !== null && nowMs - Date.parse(s.lastRunningAt) < OFF_NEWS_MS) {
|
||||
out.push({ key: `hetzner-off:${id}:${s.id}`, serverId: s.id, severity: "warning", message: `${name}: server ${s.name} is off — it was running ${ago(nowMs - Date.parse(s.lastRunningAt))} ago (Hetzner bills a server that is off)` });
|
||||
} else if (s.status !== "running" && s.status !== "off" && since >= STUCK_AFTER_MS) {
|
||||
out.push({ key: `hetzner-stuck:${id}:${s.id}`, serverId: s.id, severity: "warning", message: `${name}: server ${s.name} has been "${s.status}" for ${ago(since)}` });
|
||||
}
|
||||
if (s.rescueEnabled && s.rescueSince !== null && nowMs - Date.parse(s.rescueSince) >= RESCUE_AFTER_MS) {
|
||||
out.push({ key: `hetzner-rescue:${id}:${s.id}`, serverId: s.id, severity: "warning", message: `${name}: rescue mode is switched on for ${s.name} (${ago(nowMs - Date.parse(s.rescueSince))}) — its next reboot would come up in the rescue system, not the server` });
|
||||
}
|
||||
if (s.includedTraffic !== null && s.includedTraffic > 0 && s.outgoingTraffic !== null) {
|
||||
const used = s.outgoingTraffic / s.includedTraffic;
|
||||
if (used >= TRAFFIC_WARN) {
|
||||
out.push({
|
||||
key: `hetzner-traffic:${id}:${s.id}`,
|
||||
serverId: s.id,
|
||||
severity: used >= 1 ? "critical" : "warning",
|
||||
message: `${name}: ${s.name} has sent ${sizeText(s.outgoingTraffic)} of its ${sizeText(s.includedTraffic)} traffic this month (${Math.floor(used * 100)}%) — traffic over the quota is billed`,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (backupStaleHours > 0 && s.backupWindow !== null && images !== null && s.status !== "deleting") {
|
||||
const newest = images.filter((i) => i.type === "backup" && i.serverId === s.id && i.created).reduce<number | null>((m, i) => Math.max(m ?? 0, Date.parse(i.created as string)), null);
|
||||
const age = newest === null ? null : nowMs - newest;
|
||||
const limit = backupStaleHours * HOUR;
|
||||
const young = nowMs - Date.parse(s.created ?? "") < limit; // a server (or its backups) that is new has had no time to make one
|
||||
if (age === null ? !young : age > limit) {
|
||||
out.push({ key: `hetzner-backup:${id}:${s.id}`, serverId: s.id, severity: "warning", message: `${name}: backups are enabled for ${s.name} but ${age === null ? "none has been made" : `the newest is ${ago(age)} old`}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── Linking to the servers the app knows ───────────────────────────────────
|
||||
|
||||
/** Pure: which of the app's servers a Hetzner server is. An address wins over a name (names are chosen by hand and may differ); null when none matches. */
|
||||
export function matchAppServer(hz: Pick<HetznerServer, "name" | "ipv4" | "privateIps">, appServers: { id: number; name: string; ips: string[] }[]): { id: number; name: string } | null {
|
||||
const mine = [hz.ipv4, ...hz.privateIps].filter((i): i is string => !!i).map((i) => i.trim().toLowerCase());
|
||||
if (mine.length > 0) {
|
||||
const byIp = appServers.find((a) => a.ips.some((ip) => mine.includes(ip.trim().toLowerCase())));
|
||||
if (byIp) return { id: byIp.id, name: byIp.name };
|
||||
}
|
||||
const byName = appServers.find((a) => a.name.trim().toLowerCase() === hz.name.trim().toLowerCase());
|
||||
return byName ? { id: byName.id, name: byName.name } : null;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initHetznerSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllHetzner().catch((err) => console.error("[hetzner] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 29_000);
|
||||
console.log(`[hetzner] reading Hetzner Cloud every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -37,6 +37,8 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^hetzner-(?:off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^n8n-(?:failed):(\d+)/.exec(key);
|
||||
|
||||
@@ -5,13 +5,14 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
|
||||
import { isNextcloudFresh, nextcloudSyncStatuses } from "./nextcloudSync.js";
|
||||
import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
|
||||
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf } from "./hetznerSync.js";
|
||||
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
|
||||
import { isN8nFresh, n8nSyncStatuses, workflowsOf as n8nWorkflowsOf } from "./n8nSync.js";
|
||||
import { componentsOf as paperlessComponentsOf, failedOf as paperlessFailedOf, isPaperlessFresh, paperlessSyncStatuses } from "./paperlessSync.js";
|
||||
@@ -84,6 +85,24 @@ async function addNextcloudMetrics(out: Exposition, now: Date) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Hetzner Cloud server, for /metrics: whether it is running, its traffic this month against the quota, and the project's monthly cost estimate. Only while the reading is recent. */
|
||||
async function addHetznerMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await hetznerSyncStatuses();
|
||||
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "hetzner"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(hetznerState)) {
|
||||
const row = names.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isHetznerFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
for (const server of hetznerServersOf(s)) {
|
||||
const labels = { project: row.name, server: server.name };
|
||||
out.add("homelab_hetzner_server_running", "gauge", "1 if the Hetzner Cloud server is running, else 0.", server.status === "running" ? 1 : 0, labels);
|
||||
if (server.outgoingTraffic !== null) out.add("homelab_hetzner_traffic_outgoing_bytes", "gauge", "Outgoing traffic of a Hetzner Cloud server this month, in bytes.", server.outgoingTraffic, labels);
|
||||
if (server.includedTraffic !== null) out.add("homelab_hetzner_traffic_included_bytes", "gauge", "The outgoing traffic a Hetzner Cloud server includes per month, in bytes.", server.includedTraffic, labels);
|
||||
}
|
||||
const cost = hetznerCostOf(s);
|
||||
if (cost) out.add("homelab_hetzner_monthly_cost", "gauge", "Estimated monthly cost of a Hetzner Cloud project at list prices, including VAT, in the project's currency.", cost.total, { project: row.name, currency: cost.currency });
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Docker host's vulnerability counts from Dockhand's scans, for /metrics. Only while the reading is recent, and only for hosts whose summary could be read. */
|
||||
async function addDockerVulnMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await dockerVulnSyncStatuses();
|
||||
@@ -406,6 +425,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addNetBoxMetrics(out, now);
|
||||
await addNextcloudMetrics(out, now);
|
||||
await addDockerVulnMetrics(out, now);
|
||||
await addHetznerMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
Reference in new issue
Block a user