Add Hetzner Robot: dedicated servers and storage boxes, with shutting-down, locked and rescue-left-on alerts
Read every 15 minutes (Robot allows 200 requests an hour; read-only): each server's product, addresses, status, rescue system and paid-until, and the storage boxes. Adapter, sync service, hetzner_robot_state table, /api/hetzner-robot, Infrastructure → Hetzner Robot page, metrics and docs. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7618bf0f0a
commit
01acbaf831
31 files changed
+10105
-10
No files matched your search
@@ -91,6 +91,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
lidarr: { label: "Lidarr", link: "/media" },
|
||||
readarr: { label: "Readarr", link: "/media" },
|
||||
prowlarr: { label: "Prowlarr", link: "/media" },
|
||||
hetznerrobot: { label: "Hetzner Robot", link: "/hetzner-robot" },
|
||||
qbittorrent: { label: "qBittorrent", link: "/downloads" },
|
||||
sabnzbd: { label: "SABnzbd", link: "/downloads" },
|
||||
adguard: { label: "AdGuard Home", link: "/adguard" },
|
||||
@@ -149,6 +150,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "grafana", id: Number(m[2]), category: m[1] === "ds" ? "integration" : "monitoring" };
|
||||
m = /^(sonarr|radarr|prowlarr|lidarr|readarr)-(health|queue|disk):(\d+)/.exec(key);
|
||||
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
|
||||
m = /^hetznerrobot-(ending|rescue|setup|boxlocked|boxending):(\d+):/.exec(key);
|
||||
if (m) return { type: "hetznerrobot", id: Number(m[2]), category: "services" };
|
||||
m = /^(qbittorrent|sabnzbd)-(stuck|connection|paused|stalled|disk):(\d+)/.exec(key);
|
||||
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
|
||||
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
|
||||
@@ -305,7 +308,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr|lidarr|readarr|qbittorrent|sabnzbd|adguard|technitium|grafana):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|hetznerrobot|speedtest|fragnet|sonarr|radarr|prowlarr|lidarr|readarr|qbittorrent|sabnzbd|adguard|technitium|grafana):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] === "images" ? " (age of the images)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { downloadState, technitiumState, authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState , adguardState , grafanaState } from "../db/schema.js";
|
||||
import { hetznerRobotState, downloadState, technitiumState, authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState , adguardState , grafanaState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -40,6 +40,7 @@ import { isTechnitiumFresh, storedOf as technitiumStoredOf, technitiumProblems,
|
||||
import { grafanaProblems, grafanaSyncStatuses, isGrafanaFresh, storedOf as grafanaStoredOf, type GrafanaProblem } from "./grafanaSync.js";
|
||||
import { arrProblems, arrSyncStatuses, isArrFresh, storedOf as arrStoredOf, type ArrProblem } from "./arrSync.js";
|
||||
import { DOWNLOAD_KINDS, type DownloadKind } from "../integrations/downloads/adapter.js";
|
||||
import { isRobotFresh, robotProblems, robotSyncStatuses, storedOf as robotStoredOf, type RobotProblem } from "./hetznerRobotSync.js";
|
||||
import { downloadProblems, downloadSyncStatuses, isDownloadFresh, storedOf as downloadStoredOf, type DownloadProblem } from "./downloadsSync.js";
|
||||
import { fragnetProblems, fragnetSyncStatuses, isFragnetFresh, optionsOf as fragnetOptions, serversOf as fragnetServersOf, type FragnetProblem } from "./fragnetSync.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
@@ -144,6 +145,8 @@ export interface HealthSnapshot {
|
||||
arr?: { integrationId: number; kind: ArrKind; problems: ArrProblem[] }[];
|
||||
/** What is wrong with each qBittorrent and SABnzbd (downloads that are stuck, no connection, a queue left paused or getting nothing, a disk nearly full), only while that reading is recent (services/downloadsSync.ts). */
|
||||
downloads?: { integrationId: number; kind: DownloadKind; problems: DownloadProblem[] }[];
|
||||
/** What is wrong with each Hetzner Robot account (a server or storage box about to shut down, a locked storage box, a rescue system left on, a server stuck being set up), only while that reading is recent (services/hetznerRobotSync.ts). */
|
||||
hetznerRobot?: { integrationId: number; problems: RobotProblem[] }[];
|
||||
/** What is wrong with each AdGuard Home (DNS not running or not answering, protection left off, filter lists not updating), only while that reading is recent (services/adguardSync.ts). */
|
||||
adguard?: { integrationId: number; problems: AdguardProblem[] }[];
|
||||
/** What is wrong with each Technitium DNS Server (DNS not answering, blocking left off, block lists not updating, zones expired or not syncing), only while that reading is recent (services/technitiumSync.ts). */
|
||||
@@ -301,6 +304,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
for (const t of snapshot.technitium ?? []) for (const p of t.problems) out.push({ key: p.key, source: `technitium:${t.integrationId}`, message: p.message, severity: p.severity });
|
||||
for (const g of snapshot.grafana ?? []) for (const p of g.problems) out.push({ key: p.key, source: `grafana:${g.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// Hetzner Robot: servers and storage boxes about to shut down, a locked storage box, a rescue system left on, a server stuck being set up.
|
||||
for (const r of snapshot.hetznerRobot ?? []) for (const p of r.problems) out.push({ key: p.key, source: `hetznerrobot:${r.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// qBittorrent and SABnzbd: downloads that are stuck, no connection, a queue left paused or getting nothing, disks nearly full.
|
||||
for (const d of snapshot.downloads ?? []) for (const p of d.problems) out.push({ key: p.key, source: `${d.kind}:${d.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
@@ -693,6 +699,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
console.error("[health] couldn't judge Sonarr, Radarr and Prowlarr:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Hetzner Robot: read by its own pass every fifteen minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsH = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "hetznerrobot"), eq(integrations.enabled, true)));
|
||||
if (rowsH.length > 0) {
|
||||
const statuses = await robotSyncStatuses();
|
||||
for (const r of rowsH) {
|
||||
const [state] = await db.select().from(hetznerRobotState).where(eq(hetznerRobotState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isRobotFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`hetznerrobot:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.hetznerRobot ??= []).push({ integrationId: r.id, problems: robotProblems(r.id, r.name, robotStoredOf(state), Date.now()) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Hetzner Robot:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// qBittorrent and SABnzbd: read by their own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsD = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(and(inArray(integrations.type, DOWNLOAD_KINDS), eq(integrations.enabled, true)));
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
/**
|
||||
* Keeps what Hetzner Robot says up to date: every fifteen minutes each Robot integration is read — its dedicated servers (and whether each one's rescue system is on) and its storage boxes — and the
|
||||
* state is stored. Pages and alerts are built from what is stored. (Fifteen minutes, not five: Robot allows 200 requests an hour and a read asks about each server.)
|
||||
*
|
||||
* What is raised: **a server or storage box that is cancelled and about to shut down** (warning within 30 days, critical within 7 — the date Robot gives is the day it ends and the data goes with it);
|
||||
* **a storage box that is locked** (critical: its data cannot be reached); **a rescue system left on** for two hours (the server's next boot is into the rescue system, not its own); **a server still
|
||||
* "in process"** a day after the app first saw it so (an order that was never finished). "Since when" is what this app saw: the time of the first read that showed it.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { hetznerRobotState, integrations } from "../db/schema.js";
|
||||
import { createHetznerRobotAdapter, type RobotRead, type RobotServer, type RobotStorageBox } from "../integrations/hetznerRobot/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 15 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 45 * 60_000;
|
||||
const STATUS_FLAG = "hetznerRobotSync";
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
const DAY = 24 * HOUR;
|
||||
/** A rescue system on for this long has been forgotten. */
|
||||
export const RESCUE_AFTER_MS = 2 * HOUR;
|
||||
/** A server that has not been "ready" for this long is not being set up any more, it is stuck. */
|
||||
export const NOT_READY_AFTER_MS = DAY;
|
||||
/** A cancelled server or storage box this close to its end is news; this close, urgent. */
|
||||
export const ENDING_WARN_DAYS = 30;
|
||||
export const ENDING_CRITICAL_DAYS = 7;
|
||||
|
||||
export interface RobotSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, RobotSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, RobotSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function robotSyncStatuses(): Promise<Record<number, RobotSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isRobotFresh = (status: RobotSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof hetznerRobotState.$inferSelect;
|
||||
|
||||
function parse<T>(stored: string | null): T | null {
|
||||
if (stored === null) return null;
|
||||
try {
|
||||
return JSON.parse(stored) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** A server as stored: as Robot says it, plus what the app has seen between reads. */
|
||||
export interface StoredRobotServer extends RobotServer {
|
||||
/** When the first read that showed the rescue system on was made; null while it is off (or not known). */
|
||||
rescueSince: string | null;
|
||||
/** When the first read that showed it not "ready" was made; null while it is ready. */
|
||||
notReadySince: string | null;
|
||||
}
|
||||
|
||||
export interface StoredRobot {
|
||||
servers: StoredRobotServer[];
|
||||
storageBoxes: RobotStorageBox[] | null;
|
||||
skippedDetail: number;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export const storedOf = (row: StateRow): StoredRobot => ({
|
||||
servers: (parse<StoredRobotServer[]>(row.servers) ?? []).map((s) => ({ ...s, rescueSince: s.rescueSince ?? null, notReadySince: s.notReadySince ?? null })),
|
||||
storageBoxes: parse<RobotStorageBox[]>(row.storageBoxes),
|
||||
skippedDetail: row.skippedDetail,
|
||||
updatedAt: row.updatedAt,
|
||||
});
|
||||
|
||||
/** Pure: the servers as read, with what was seen between reads carried over from the last read by server number. A thing first seen wrong starts its clock now. */
|
||||
export function carrySince(servers: RobotServer[], previous: StoredRobotServer[] | null, now: Date): StoredRobotServer[] {
|
||||
const at = now.toISOString();
|
||||
const before = new Map((previous ?? []).map((s) => [s.number, s]));
|
||||
return servers.map((s) => ({
|
||||
...s,
|
||||
rescueSince: s.rescue === true ? (before.get(s.number)?.rescueSince ?? at) : null,
|
||||
notReadySince: s.status !== "ready" ? (before.get(s.number)?.notReadySince ?? at) : null,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
|
||||
export async function storeRead(integrationId: number, r: RobotRead, now: Date = new Date()): Promise<void> {
|
||||
const [previous] = await db.select().from(hetznerRobotState).where(eq(hetznerRobotState.integrationId, integrationId)).limit(1);
|
||||
const values = {
|
||||
servers: JSON.stringify(carrySince(r.servers, previous ? storedOf(previous).servers : null, now)),
|
||||
storageBoxes: r.storageBoxes && JSON.stringify(r.storageBoxes),
|
||||
skippedDetail: r.skippedDetail,
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(hetznerRobotState).values({ integrationId, ...values }).onConflictDoUpdate({ target: hetznerRobotState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one Robot account and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncRobot(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<RobotRead>): Promise<RobotSyncStatus> {
|
||||
let status: RobotSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "hetznerrobot") throw new Error("not a Hetzner Robot integration");
|
||||
const result = await (read ? read(loaded.config) : createHetznerRobotAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[hetzner-robot] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[hetzner-robot] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllRobot(now: Date = new Date()): Promise<Record<number, RobotSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "hetznerrobot"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, RobotSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncRobot(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetRobot(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface RobotProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
|
||||
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / DAY), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
|
||||
|
||||
/** Pure: how many calendar days (UTC) there are from today to a date "YYYY-MM-DD": 0 on the day itself, which it still runs through, and negative once it has passed. */
|
||||
export function daysLeft(date: string | null, nowMs: number): number | null {
|
||||
if (date === null) return null;
|
||||
const end = Date.parse(date);
|
||||
if (!Number.isFinite(end)) return null;
|
||||
const now = new Date(nowMs);
|
||||
return Math.round((end - Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / DAY);
|
||||
}
|
||||
|
||||
const endsIn = (days: number) => (days < 0 ? "has ended" : days === 0 ? "ends today" : days === 1 ? "ends tomorrow" : `ends in ${days} days`);
|
||||
|
||||
/** Pure: what is wrong with one Robot account, from what is stored for it. */
|
||||
export function robotProblems(id: number, name: string, s: StoredRobot, nowMs: number): RobotProblem[] {
|
||||
const out: RobotProblem[] = [];
|
||||
for (const v of s.servers) {
|
||||
const label = `${v.name} (server ${v.number})`;
|
||||
if (v.cancelled) {
|
||||
const left = daysLeft(v.paidUntil, nowMs);
|
||||
if (left !== null && left <= ENDING_WARN_DAYS) {
|
||||
out.push({ key: `hetznerrobot-ending:${id}:${v.number}`, severity: left <= ENDING_CRITICAL_DAYS ? "critical" : "warning", message: `${name}: ${label} is cancelled and ${endsIn(left)} (${v.paidUntil}) — the server and its data go with it` });
|
||||
}
|
||||
}
|
||||
if (v.rescue === true && v.rescueSince !== null && nowMs - Date.parse(v.rescueSince) >= RESCUE_AFTER_MS) {
|
||||
out.push({ key: `hetznerrobot-rescue:${id}:${v.number}`, severity: "warning", message: `${name}: the rescue system of ${label} has been switched on for ${ago(nowMs - Date.parse(v.rescueSince))} — its next boot is into the rescue system, not its own` });
|
||||
}
|
||||
if (v.status !== "ready" && v.notReadySince !== null && nowMs - Date.parse(v.notReadySince) >= NOT_READY_AFTER_MS) {
|
||||
out.push({ key: `hetznerrobot-setup:${id}:${v.number}`, severity: "warning", message: `${name}: ${label} has been “${v.status}” for ${ago(nowMs - Date.parse(v.notReadySince))} instead of ready` });
|
||||
}
|
||||
}
|
||||
for (const b of s.storageBoxes ?? []) {
|
||||
const label = `storage box ${b.name}${b.login ? ` (${b.login})` : ""}`;
|
||||
if (b.locked) out.push({ key: `hetznerrobot-boxlocked:${id}:${b.id}`, severity: "critical", message: `${name}: ${label} is locked — its data cannot be reached` });
|
||||
if (b.cancelled) {
|
||||
const left = daysLeft(b.paidUntil, nowMs);
|
||||
if (left !== null && left <= ENDING_WARN_DAYS) {
|
||||
out.push({ key: `hetznerrobot-boxending:${id}:${b.id}`, severity: left <= ENDING_CRITICAL_DAYS ? "critical" : "warning", message: `${name}: ${label} is cancelled and ${endsIn(left)} (${b.paidUntil}) — its data goes with it` });
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initHetznerRobotSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllRobot().catch((err) => console.error("[hetzner-robot] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 47_000);
|
||||
console.log(`[hetzner-robot] reading Hetzner Robot every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
* and not the other is caught. A kind in no group (an older one) is simply not in any group's summary.
|
||||
*/
|
||||
export const INTEGRATION_GROUPS: { id: string; label: string; types: string[] }[] = [
|
||||
{ id: "infrastructure", label: "Infrastructure", types: ["proxmox", "synology", "dockhand", "hetzner", "fragnet", "tailscale"] },
|
||||
{ id: "infrastructure", label: "Infrastructure", types: ["proxmox", "synology", "dockhand", "hetzner", "hetznerrobot", "fragnet", "tailscale"] },
|
||||
{ id: "apps", label: "Apps", types: ["homeassistant", "jellyfin", "sonarr", "radarr", "lidarr", "readarr", "prowlarr", "qbittorrent", "sabnzbd", "immich", "paperless", "nextcloud", "authentik"] },
|
||||
{ id: "backups", label: "Backups", types: ["pbs"] },
|
||||
{ id: "network", label: "Network", types: ["unifi", "npm", "phpipam", "netbox", "adguard", "technitium"] },
|
||||
|
||||
@@ -45,6 +45,8 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^(?:sonarr|radarr|prowlarr|lidarr|readarr)-(?:health|queue|disk):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^hetznerrobot-(?:ending|rescue|setup|boxlocked|boxending):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^(?:qbittorrent|sabnzbd)-(?:stuck|connection|paused|stalled|disk):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^version-behind:(\d+)$/.exec(key);
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
import { eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { downloadState, technitiumState, assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState, adguardState, grafanaState } from "../db/schema.js";
|
||||
import { hetznerRobotState, downloadState, technitiumState, assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState, adguardState, grafanaState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
@@ -15,6 +15,7 @@ import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
|
||||
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
|
||||
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf, loadBalancersOf as hetznerLoadBalancersOf, certificatesOf as hetznerCertificatesOf, trafficForecast as hetznerTrafficForecast } from "./hetznerSync.js";
|
||||
import { ARR_KINDS } from "../integrations/arr/adapter.js";
|
||||
import { daysLeft as robotDaysLeft, isRobotFresh, robotSyncStatuses, storedOf as robotStoredOf } from "./hetznerRobotSync.js";
|
||||
import { DOWNLOAD_KINDS, DOWNLOAD_STATES } from "../integrations/downloads/adapter.js";
|
||||
import { downloadSyncStatuses, isDownloadFresh, recentFailed as downloadsFailed, storedOf as downloadStoredOf, stuckItems as downloadStuck } from "./downloadsSync.js";
|
||||
import { versionRows as appVersionRows } from "./appVersions.js";
|
||||
@@ -212,6 +213,34 @@ async function addArrMetrics(out: Exposition, now: Date) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Hetzner Robot account, for /metrics: for each dedicated server whether its rescue system is on and, if it is cancelled, the days left; for each storage box whether it is locked and, if cancelled, the days left. Only while the reading is recent. */
|
||||
async function addRobotMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await robotSyncStatuses();
|
||||
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "hetznerrobot"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(hetznerRobotState)) {
|
||||
const row = rows.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isRobotFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const state = robotStoredOf(s);
|
||||
out.add("homelab_hetzner_robot_servers", "gauge", "How many dedicated servers the Hetzner Robot account has.", state.servers.length, { name: row.name });
|
||||
for (const v of state.servers) {
|
||||
const labels = { name: row.name, server: String(v.number), server_name: v.name };
|
||||
if (v.rescue !== null) out.add("homelab_hetzner_robot_rescue_active", "gauge", "1 if a dedicated server's rescue system is switched on for the next boot, else 0.", v.rescue ? 1 : 0, labels);
|
||||
if (v.cancelled) {
|
||||
const left = robotDaysLeft(v.paidUntil, now.getTime());
|
||||
if (left !== null) out.add("homelab_hetzner_robot_days_left", "gauge", "Calendar days left of a cancelled dedicated server or storage box: 0 on the day it ends, negative after.", left, labels);
|
||||
}
|
||||
}
|
||||
for (const b of state.storageBoxes ?? []) {
|
||||
const labels = { name: row.name, storage_box: String(b.id), storage_box_name: b.name };
|
||||
out.add("homelab_hetzner_robot_storage_box_locked", "gauge", "1 if a storage box is locked, else 0.", b.locked ? 1 : 0, labels);
|
||||
if (b.cancelled) {
|
||||
const left = robotDaysLeft(b.paidUntil, now.getTime());
|
||||
if (left !== null) out.add("homelab_hetzner_robot_days_left", "gauge", "Calendar days left of a cancelled dedicated server or storage box: 0 on the day it ends, negative after.", left, labels);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Each qBittorrent and SABnzbd, for /metrics: how fast it is going, the items in each state and how many are stuck, whether it is connected or paused, what failed in the last day and the free space of its disk. Only while the reading is recent. */
|
||||
async function addDownloadMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await downloadSyncStatuses();
|
||||
@@ -609,6 +638,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addFragnetMetrics(out, now);
|
||||
await addArrMetrics(out, now);
|
||||
await addDownloadMetrics(out, now);
|
||||
await addRobotMetrics(out, now);
|
||||
await addAdguardMetrics(out, now);
|
||||
await addTechnitiumMetrics(out, now);
|
||||
await addVersionMetrics(out);
|
||||
|
||||
Reference in new issue
Block a user