Sonarr/Radarr/Prowlarr, Hetzner load balancers/floating IPs/certificates, traffic forecast
New integration kinds sonarr, radarr and prowlarr (shared adapter and sync, page under Apps -> Media managers): own health findings, stuck downloads, nearly full disks. Hetzner now reads load balancers (target health), floating IPs and certificates, includes them in the cost estimate, and forecasts the month's traffic against the quota. Migrations 0072 and 0073; docs, metrics and tests included. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9a3d5c466a
commit
d9083adb5b
38 files changed
+18374
-42
No files matched your search
@@ -86,6 +86,9 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
hetzner: { label: "Hetzner", link: "/hetzner" },
|
||||
speedtest: { label: "Speedtest", link: "/speedtest" },
|
||||
fragnet: { label: "Fragnet", link: "/game-servers" },
|
||||
sonarr: { label: "Sonarr", link: "/media" },
|
||||
radarr: { label: "Radarr", link: "/media" },
|
||||
prowlarr: { label: "Prowlarr", link: "/media" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -126,8 +129,10 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
|
||||
if (m) return { type: "speedtest", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^hetzner-(off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
|
||||
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
|
||||
m = /^hetzner-(off|stuck|rescue|traffic|forecast|backup|lb|cert|fip):(\d+):/.exec(key);
|
||||
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" || m[1] === "forecast" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
|
||||
m = /^(sonarr|radarr|prowlarr)-(health|queue|disk):(\d+)/.exec(key);
|
||||
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
|
||||
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
|
||||
if (m) return { type: "fragnet", id: Number(m[2]), category: m[1] === "disk" ? "disk" : m[1] === "backup" ? "backup" : "monitoring" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
@@ -270,7 +275,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : origin.type === "sonarr" || origin.type === "radarr" || origin.type === "prowlarr" ? "/media" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
@@ -280,7 +285,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet|sonarr|radarr|prowlarr):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
/**
|
||||
* Keeps what Sonarr, Radarr and Prowlarr say up to date: every five minutes each one is read — what its own checks found wrong, its download queue, its disks, how many wanted items are
|
||||
* missing and (Prowlarr) its indexers — and the state is stored. Pages and alerts are built from what is stored, so none of them talks to the applications.
|
||||
*
|
||||
* What is raised: **what the application itself says is wrong** (its health checks of type warning or error: a download client that cannot be reached, indexers switched off after failures, a
|
||||
* root folder gone — a mere notice, such as an update being available, is not raised); **downloads that are stuck** — in the queue for two hours or more and judged warning or error, or failed,
|
||||
* or blocked from importing (one alert for the lot, naming the first few); **a disk nearly full** (under 10% free, critical under 5%). The same three for each of the three, under one key each.
|
||||
*/
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { arrState, integrations } from "../db/schema.js";
|
||||
import { ARR_KINDS, ARR_NAMES, createArrAdapter, type ArrDisk, type ArrHealthIssue, type ArrIndexer, type ArrKind, type ArrQueueItem, type ArrRead } from "../integrations/arr/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
const STATUS_FLAG = "arrSync";
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
/** A download judged wrong for less than this is left to sort itself out (a client that is just slow to answer). */
|
||||
export const STUCK_AFTER_MS = 2 * HOUR;
|
||||
export const DISK_WARN_FREE = 0.1;
|
||||
export const DISK_CRITICAL_FREE = 0.05;
|
||||
/** A disk smaller than this is not a media disk (a tmpfs, a boot partition). */
|
||||
export const MIN_DISK_BYTES = 2 * 1024 ** 3;
|
||||
|
||||
export interface ArrSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, ArrSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, ArrSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function arrSyncStatuses(): Promise<Record<number, ArrSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isArrFresh = (status: ArrSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof arrState.$inferSelect;
|
||||
|
||||
function parse<T>(stored: string | null): T | null {
|
||||
if (stored === null) return null;
|
||||
try {
|
||||
return JSON.parse(stored) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** What is stored of one application, as the read gave it. */
|
||||
export interface StoredArr {
|
||||
version: string | null;
|
||||
health: ArrHealthIssue[];
|
||||
queue: { total: number; items: ArrQueueItem[] } | null;
|
||||
disks: ArrDisk[] | null;
|
||||
missing: number | null;
|
||||
indexers: ArrIndexer[] | null;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export const storedOf = (row: StateRow): StoredArr => ({
|
||||
version: row.version,
|
||||
health: parse<ArrHealthIssue[]>(row.health) ?? [],
|
||||
queue: parse<{ total: number; items: ArrQueueItem[] }>(row.queue),
|
||||
disks: parse<ArrDisk[]>(row.disks),
|
||||
missing: row.missing,
|
||||
indexers: parse<ArrIndexer[]>(row.indexers),
|
||||
updatedAt: row.updatedAt,
|
||||
});
|
||||
|
||||
/** Writes down one read, replacing the last. */
|
||||
export async function storeRead(integrationId: number, r: ArrRead, now: Date = new Date()): Promise<void> {
|
||||
const values = {
|
||||
version: r.version,
|
||||
health: JSON.stringify(r.health),
|
||||
queue: r.queue && JSON.stringify(r.queue),
|
||||
disks: r.disks && JSON.stringify(r.disks),
|
||||
missing: r.missing,
|
||||
indexers: r.indexers && JSON.stringify(r.indexers),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(arrState).values({ integrationId, ...values }).onConflictDoUpdate({ target: arrState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one application and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncArr(integrationId: number, now: Date = new Date(), read?: (kind: ArrKind, config: Record<string, unknown>) => Promise<ArrRead>): Promise<ArrSyncStatus> {
|
||||
let status: ArrSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
const kind = loaded?.integration.type as ArrKind | undefined;
|
||||
if (!loaded || !kind || !ARR_KINDS.includes(kind)) throw new Error("not a Sonarr, Radarr or Prowlarr integration");
|
||||
const result = await (read ? read(kind, loaded.config) : createArrAdapter(kind, loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[arr] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[arr] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllArr(now: Date = new Date()): Promise<Record<number, ArrSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(inArray(integrations.type, ARR_KINDS), eq(integrations.enabled, true)));
|
||||
const out: Record<number, ArrSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncArr(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetArr(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface ArrProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const slug = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40);
|
||||
|
||||
/** A short stable stamp of a message, numbers left out (so "3 indexers" and "4 indexers" are the same finding). */
|
||||
function stamp(message: string): string {
|
||||
let h = 0;
|
||||
for (const c of message.replace(/\d+/g, "#")) h = (h * 31 + c.charCodeAt(0)) >>> 0;
|
||||
return h.toString(36);
|
||||
}
|
||||
|
||||
export const sizeText = (bytes: number): string => {
|
||||
const gb = bytes / 1024 ** 3;
|
||||
return gb >= 1024 ? `${(gb / 1024).toFixed(1)} TB` : `${gb.toFixed(gb >= 100 ? 0 : 1)} GB`;
|
||||
};
|
||||
|
||||
/** A finding starts with a capital as a sentence of its own; after "says" it reads better without ("No download client" → "no download client"), unless it starts with a name or an acronym. */
|
||||
const lcFirst = (s: string) => (/^[A-Z][a-z]/.test(s) ? s[0].toLowerCase() + s.slice(1) : s);
|
||||
|
||||
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
|
||||
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / (24 * HOUR)), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
|
||||
|
||||
/** Pure: whether a queue item is stuck — judged wrong, or failed, or blocked from importing — and has been in the queue long enough for it not to sort itself out. */
|
||||
export function isStuck(i: ArrQueueItem, nowMs: number): boolean {
|
||||
const bad = i.tracked === "warning" || i.tracked === "error" || i.state === "failed" || i.state === "failedPending" || i.state === "importBlocked" || i.status === "failed";
|
||||
if (!bad) return false;
|
||||
const since = i.added === null ? null : nowMs - Date.parse(i.added);
|
||||
return since !== null && since >= STUCK_AFTER_MS;
|
||||
}
|
||||
|
||||
/** Pure: the stuck items, longest in the queue first. */
|
||||
export const stuckItems = (queue: StoredArr["queue"], nowMs: number): ArrQueueItem[] => (queue?.items ?? []).filter((i) => isStuck(i, nowMs)).sort((a, b) => Date.parse(a.added ?? "") - Date.parse(b.added ?? ""));
|
||||
|
||||
/** Pure: the disks worth judging — one for each filesystem (the same size and free space is the same disk, however many folders are on it). */
|
||||
export function distinctDisks(disks: ArrDisk[] | null): ArrDisk[] {
|
||||
const seen = new Set<string>();
|
||||
const out: ArrDisk[] = [];
|
||||
for (const d of [...(disks ?? [])].sort((a, b) => a.path.length - b.path.length || a.path.localeCompare(b.path))) {
|
||||
if (d.total < MIN_DISK_BYTES) continue;
|
||||
const id = `${d.total}:${d.free}`;
|
||||
if (seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
out.push(d);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: what is wrong with one application, from what is stored for it. */
|
||||
export function arrProblems(kind: ArrKind, id: number, name: string, state: StoredArr, nowMs: number): ArrProblem[] {
|
||||
const out: ArrProblem[] = [];
|
||||
const app = ARR_NAMES[kind];
|
||||
for (const h of state.health) {
|
||||
if (h.type !== "warning" && h.type !== "error") continue;
|
||||
out.push({ key: `${kind}-health:${id}:${slug(h.source)}-${stamp(h.message)}`, severity: "warning", message: `${name}: ${app} says ${lcFirst(h.message.trim())}` });
|
||||
}
|
||||
const stuck = stuckItems(state.queue, nowMs);
|
||||
if (stuck.length > 0) {
|
||||
const names = stuck.slice(0, 3).map((i) => i.title);
|
||||
const oldest = stuck[0].added === null ? null : nowMs - Date.parse(stuck[0].added);
|
||||
const why = stuck[0].messages[0];
|
||||
out.push({
|
||||
key: `${kind}-queue:${id}`,
|
||||
severity: "warning",
|
||||
message: `${name}: ${plural(stuck.length, "download")} ${stuck.length === 1 ? "is" : "are"} stuck — ${names.join("; ")}${stuck.length > 3 ? `; and ${stuck.length - 3} more` : ""}${oldest !== null ? ` (the oldest for ${ago(oldest)})` : ""}${why ? ` — ${why}` : ""}`,
|
||||
});
|
||||
}
|
||||
for (const d of distinctDisks(state.disks)) {
|
||||
const free = d.free / d.total;
|
||||
if (free < DISK_WARN_FREE) {
|
||||
out.push({ key: `${kind}-disk:${id}:${slug(d.path)}`, severity: free < DISK_CRITICAL_FREE ? "critical" : "warning", message: `${name}: ${d.path} has ${sizeText(d.free)} free of ${sizeText(d.total)} (${Math.floor(free * 100)}%)` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initArrSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllArr().catch((err) => console.error("[arr] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 37_000);
|
||||
console.log(`[arr] reading Sonarr, Radarr and Prowlarr every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState } from "../db/schema.js";
|
||||
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState , arrState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -30,7 +30,9 @@ import { isNetBoxFresh, netboxProblems, netboxSyncStatuses, type NetBoxProblem }
|
||||
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type NextcloudProblem } from "./nextcloudSync.js";
|
||||
import { dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, type DockerVulnProblem } from "./dockerVulnSync.js";
|
||||
import { expectedOf as speedtestExpected, isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestProblems, speedtestSyncStatuses, type SpeedtestProblem } from "./speedtestSync.js";
|
||||
import { hetznerProblems, hetznerSyncStatuses, imagesOf as hetznerImagesOf, isHetznerFresh, serversOf as hetznerServersOf, type HetznerProblem } from "./hetznerSync.js";
|
||||
import { hetznerProblems, hetznerSyncStatuses, imagesOf as hetznerImagesOf, loadBalancersOf as hetznerLoadBalancersOf, floatingIpsOf as hetznerFloatingIpsOf, certificatesOf as hetznerCertificatesOf, isHetznerFresh, serversOf as hetznerServersOf, type HetznerProblem } from "./hetznerSync.js";
|
||||
import { ARR_KINDS, type ArrKind } from "../integrations/arr/adapter.js";
|
||||
import { arrProblems, arrSyncStatuses, isArrFresh, storedOf as arrStoredOf, type ArrProblem } from "./arrSync.js";
|
||||
import { fragnetProblems, fragnetSyncStatuses, isFragnetFresh, optionsOf as fragnetOptions, serversOf as fragnetServersOf, type FragnetProblem } from "./fragnetSync.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
@@ -128,6 +130,8 @@ export interface HealthSnapshot {
|
||||
speedtest?: { integrationId: number; problems: SpeedtestProblem[] }[];
|
||||
/** What is wrong with each Fragnet game panel account (a backup that is late, a disk or memory nearly full), only while that reading is recent (services/fragnetSync.ts). */
|
||||
fragnet?: { integrationId: number; problems: FragnetProblem[] }[];
|
||||
/** What is wrong with each Sonarr, Radarr and Prowlarr (what it says itself, downloads that are stuck, a disk nearly full), only while that reading is recent (services/arrSync.ts). */
|
||||
arr?: { integrationId: number; kind: ArrKind; problems: ArrProblem[] }[];
|
||||
/** Containers to leave alone. */
|
||||
dockerIgnore?: string[];
|
||||
/** The clock the container judgements are made on (the pass's, so a test can set it). */
|
||||
@@ -269,6 +273,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
// Speedtest Tracker: the internet slower than it is sold at, speed tests that keep failing.
|
||||
for (const s of snapshot.speedtest ?? []) for (const p of s.problems) out.push({ key: p.key, source: `speedtest:${s.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// Sonarr, Radarr and Prowlarr: what they say is wrong, downloads that are stuck, disks nearly full.
|
||||
for (const a of snapshot.arr ?? []) for (const p of a.problems) out.push({ key: p.key, source: `${a.kind}:${a.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// Fragnet game servers: backups that are late, disks and memory nearly full.
|
||||
for (const f of snapshot.fragnet ?? []) for (const p of f.problems) out.push({ key: p.key, source: `fragnet:${f.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
@@ -537,6 +544,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
console.error("[health] couldn't judge Speedtest Tracker:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Sonarr, Radarr and Prowlarr: read by their own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsA = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(and(inArray(integrations.type, ARR_KINDS), eq(integrations.enabled, true)));
|
||||
if (rowsA.length > 0) {
|
||||
const statuses = await arrSyncStatuses();
|
||||
for (const r of rowsA) {
|
||||
const [state] = await db.select().from(arrState).where(eq(arrState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isArrFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`${r.type}:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.arr ??= []).push({ integrationId: r.id, kind: r.type as ArrKind, problems: arrProblems(r.type as ArrKind, r.id, r.name, arrStoredOf(state), Date.now()) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Sonarr, Radarr and Prowlarr:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Fragnet game servers: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsF = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "fragnet"), eq(integrations.enabled, true)));
|
||||
@@ -567,7 +592,7 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
held.add(`hetzner:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.hetzner ??= []).push({ integrationId: r.id, problems: hetznerProblems(r.id, r.name, hetznerServersOf(state), hetznerImagesOf(state), Date.now(), checks.backupStaleHours) });
|
||||
(snapshot.hetzner ??= []).push({ integrationId: r.id, problems: hetznerProblems(r.id, r.name, hetznerServersOf(state), hetznerImagesOf(state), Date.now(), checks.backupStaleHours, { loadBalancers: hetznerLoadBalancersOf(state), floatingIps: hetznerFloatingIpsOf(state), certificates: hetznerCertificatesOf(state) }) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { hetznerState, integrations } from "../db/schema.js";
|
||||
import { createHetznerAdapter, type HetznerImage, type HetznerRead, type HetznerServer, type HetznerVolume } from "../integrations/hetzner/adapter.js";
|
||||
import { createHetznerAdapter, type HetznerCertificate, type HetznerFloatingIp, type HetznerImage, type HetznerLoadBalancer, type HetznerRead, type HetznerServer, type HetznerVolume } from "../integrations/hetzner/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
@@ -28,6 +28,17 @@ export const STUCK_AFTER_MS = 30 * MIN;
|
||||
/** Rescue mode switched on and not used within this long: the next reboot would come up in it. */
|
||||
export const RESCUE_AFTER_MS = HOUR;
|
||||
export const TRAFFIC_WARN = 0.8;
|
||||
const DAY = 24 * HOUR;
|
||||
/** A certificate this close to its end is a warning; this close (or past it) is critical. */
|
||||
export const CERT_WARN_DAYS = 14;
|
||||
export const CERT_CRITICAL_DAYS = 3;
|
||||
/** The traffic of a server is written down about this often (more reads than that are not kept), for the last two weeks. */
|
||||
export const TRAFFIC_SAMPLE_EVERY_MS = 6 * HOUR;
|
||||
export const TRAFFIC_SAMPLE_KEEP_MS = 14 * DAY;
|
||||
/** A forecast needs this much of the month behind it (by the month's average), or this much of the last week (by the recent rate). */
|
||||
export const FORECAST_MIN_MONTH_MS = 3 * DAY;
|
||||
export const FORECAST_MIN_RECENT_MS = 2 * DAY;
|
||||
export const FORECAST_RECENT_WINDOW_MS = 7 * DAY;
|
||||
|
||||
export interface HetznerSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
@@ -61,6 +72,13 @@ export interface StoredServer extends HetznerServer {
|
||||
statusSince: string;
|
||||
/** When the first read that showed rescue mode on was made; null when it is not on. */
|
||||
rescueSince: string | null;
|
||||
/** The month's outgoing traffic as it was now and then (about every six hours, for two weeks, starting over when the counter goes down — a new month); what the forecast is made from. */
|
||||
trafficHistory?: TrafficSample[];
|
||||
}
|
||||
|
||||
export interface TrafficSample {
|
||||
at: string;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export interface HetznerCost {
|
||||
@@ -70,6 +88,9 @@ export interface HetznerCost {
|
||||
volumes: number;
|
||||
snapshots: number;
|
||||
ips: number;
|
||||
/** Load balancers and floating IPs (absent in an estimate stored before they were read). */
|
||||
loadBalancers?: number;
|
||||
floatingIps?: number;
|
||||
total: number;
|
||||
/** Primary IPs that belong to no server (billed all the same). */
|
||||
unassignedIps: number;
|
||||
@@ -90,6 +111,9 @@ const parseList = <T>(stored: string | null): T[] | null => {
|
||||
export const serversOf = (row: Pick<StateRow, "servers">): StoredServer[] => parseList<StoredServer>(row.servers) ?? [];
|
||||
export const volumesOf = (row: Pick<StateRow, "volumes">): HetznerVolume[] | null => parseList<HetznerVolume>(row.volumes);
|
||||
export const imagesOf = (row: Pick<StateRow, "images">): HetznerImage[] | null => parseList<HetznerImage>(row.images);
|
||||
export const loadBalancersOf = (row: Pick<StateRow, "loadBalancers">): HetznerLoadBalancer[] | null => parseList<HetznerLoadBalancer>(row.loadBalancers);
|
||||
export const floatingIpsOf = (row: Pick<StateRow, "floatingIps">): HetznerFloatingIp[] | null => parseList<HetznerFloatingIp>(row.floatingIps);
|
||||
export const certificatesOf = (row: Pick<StateRow, "certificates">): HetznerCertificate[] | null => parseList<HetznerCertificate>(row.certificates);
|
||||
export const costOf = (row: Pick<StateRow, "cost">): HetznerCost | null => {
|
||||
if (!row.cost) return null;
|
||||
try {
|
||||
@@ -99,6 +123,55 @@ export const costOf = (row: Pick<StateRow, "cost">): HetznerCost | null => {
|
||||
}
|
||||
};
|
||||
|
||||
/** Pure: the traffic history with this read added (one sample about every six hours); a counter that went down is a new month, so the history starts over; old samples are dropped. */
|
||||
export function carryTraffic(bytes: number | null, history: TrafficSample[], now: Date): TrafficSample[] {
|
||||
if (bytes === null) return history;
|
||||
const nowMs = now.getTime();
|
||||
let kept = history.filter((h) => nowMs - Date.parse(h.at) <= TRAFFIC_SAMPLE_KEEP_MS);
|
||||
const last = kept[kept.length - 1];
|
||||
if (last && bytes < last.bytes) kept = [];
|
||||
const newest = kept[kept.length - 1];
|
||||
if (!newest || nowMs - Date.parse(newest.at) >= TRAFFIC_SAMPLE_EVERY_MS) kept = [...kept, { at: now.toISOString(), bytes }];
|
||||
return kept;
|
||||
}
|
||||
|
||||
export interface TrafficForecast {
|
||||
/** Bytes a day, the way it has been going. */
|
||||
perDay: number;
|
||||
/** What the month will have come to by its end at that rate. */
|
||||
projectedMonth: number;
|
||||
/** When the quota is passed, if it is before the month ends; null when it is not (or already has been). */
|
||||
exhaustedAt: string | null;
|
||||
/** Whether the rate is the last week's or the month's average. */
|
||||
basis: "recent" | "month";
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: how the month's traffic is going to end, by the rate of the last week (or, with too little of it, the month's average so far) — null when there is too little to say anything. The
|
||||
* month is taken to be the calendar month in UTC, which is how Hetzner counts it (not checked against a live project: a counter that goes down starts the history over anyway).
|
||||
*/
|
||||
export function trafficForecast(s: Pick<StoredServer, "outgoingTraffic" | "includedTraffic" | "trafficHistory">, nowMs: number): TrafficForecast | null {
|
||||
const used = s.outgoingTraffic;
|
||||
const included = s.includedTraffic;
|
||||
if (used === null || included === null || included <= 0 || used <= 0) return null;
|
||||
const d = new Date(nowMs);
|
||||
const monthStart = Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), 1);
|
||||
const monthEnd = Date.UTC(d.getUTCFullYear(), d.getUTCMonth() + 1, 1);
|
||||
let perDay: number | null = null;
|
||||
let basis: TrafficForecast["basis"] = "recent";
|
||||
const window = (s.trafficHistory ?? []).filter((h) => Date.parse(h.at) >= Math.max(monthStart, nowMs - FORECAST_RECENT_WINDOW_MS) && h.bytes <= used);
|
||||
const oldest = window[0];
|
||||
if (oldest && nowMs - Date.parse(oldest.at) >= FORECAST_MIN_RECENT_MS) perDay = ((used - oldest.bytes) / (nowMs - Date.parse(oldest.at))) * DAY;
|
||||
if (perDay === null && nowMs - monthStart >= FORECAST_MIN_MONTH_MS) {
|
||||
perDay = (used / (nowMs - monthStart)) * DAY;
|
||||
basis = "month";
|
||||
}
|
||||
if (perDay === null) return null;
|
||||
const projectedMonth = used + (perDay * (monthEnd - nowMs)) / DAY;
|
||||
const exhaustedAt = used < included && projectedMonth > included && perDay > 0 ? new Date(nowMs + ((included - used) / perDay) * DAY).toISOString() : null;
|
||||
return { perDay, projectedMonth, exhaustedAt, basis };
|
||||
}
|
||||
|
||||
/** Pure: what the app has seen between reads, carried from the last read of the same server. A server first seen starts its clocks now. */
|
||||
export function carryTimes(servers: HetznerServer[], previous: StoredServer[], now: Date): StoredServer[] {
|
||||
const before = new Map(previous.map((p) => [p.id, p]));
|
||||
@@ -107,6 +180,7 @@ export function carryTimes(servers: HetznerServer[], previous: StoredServer[], n
|
||||
const p = before.get(s.id);
|
||||
return {
|
||||
...s,
|
||||
trafficHistory: carryTraffic(s.outgoingTraffic, p?.trafficHistory ?? [], now),
|
||||
lastRunningAt: s.status === "running" ? at : (p?.lastRunningAt ?? null),
|
||||
statusSince: p && p.status === s.status ? p.statusSince : at,
|
||||
rescueSince: s.rescueEnabled ? (p?.rescueEnabled ? (p.rescueSince ?? at) : at) : null,
|
||||
@@ -115,7 +189,7 @@ export function carryTimes(servers: HetznerServer[], previous: StoredServer[], n
|
||||
}
|
||||
|
||||
/** Pure: roughly what the project costs per month at Hetzner's list prices, including VAT; null when the price list could not be read. Load balancers, floating IPs and traffic over the quota are not in it. */
|
||||
export function estimateCost(read: Pick<HetznerRead, "servers" | "volumes" | "images" | "primaryIps" | "prices">): HetznerCost | null {
|
||||
export function estimateCost(read: Pick<HetznerRead, "servers" | "volumes" | "images" | "primaryIps" | "prices"> & Partial<Pick<HetznerRead, "loadBalancers" | "floatingIps">>): HetznerCost | null {
|
||||
const p = read.prices;
|
||||
if (!p) return null;
|
||||
let servers = 0;
|
||||
@@ -137,9 +211,21 @@ export function estimateCost(read: Pick<HetznerRead, "servers" | "volumes" | "im
|
||||
ips += p.primaryIps.find((x) => x.type === ip.type && x.location === ip.location)?.monthly ?? 0;
|
||||
if (!ip.assigned) unassignedIps++;
|
||||
}
|
||||
let loadBalancers = 0;
|
||||
for (const lb of read.loadBalancers ?? []) {
|
||||
if (lb.monthlyPrice === null) unpriced++;
|
||||
else loadBalancers += lb.monthlyPrice;
|
||||
}
|
||||
let floatingIps = 0;
|
||||
for (const f of read.floatingIps ?? []) {
|
||||
const prices = p.floatingIps ?? [];
|
||||
const found = prices.find((x) => x.type === f.type && x.location === f.location) ?? prices.find((x) => x.type === f.type && x.location === "*");
|
||||
if (found) floatingIps += found.monthly;
|
||||
else unpriced++;
|
||||
}
|
||||
const round = (n: number) => Math.round(n * 100) / 100;
|
||||
const parts = { servers: round(servers), backups: round(backups), volumes: round(volumes), snapshots: round(snapshots), ips: round(ips) };
|
||||
return { currency: p.currency, ...parts, total: round(servers + backups + volumes + snapshots + ips), unassignedIps, unpriced };
|
||||
const parts = { servers: round(servers), backups: round(backups), volumes: round(volumes), snapshots: round(snapshots), ips: round(ips), loadBalancers: round(loadBalancers), floatingIps: round(floatingIps) };
|
||||
return { currency: p.currency, ...parts, total: round(servers + backups + volumes + snapshots + ips + loadBalancers + floatingIps), unassignedIps, unpriced };
|
||||
}
|
||||
|
||||
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
|
||||
@@ -149,6 +235,9 @@ export async function storeRead(integrationId: number, r: HetznerRead, now: Date
|
||||
servers: JSON.stringify(carryTimes(r.servers, previous ? serversOf(previous) : [], now)),
|
||||
volumes: r.volumes && JSON.stringify(r.volumes),
|
||||
images: r.images && JSON.stringify(r.images),
|
||||
loadBalancers: r.loadBalancers && JSON.stringify(r.loadBalancers),
|
||||
floatingIps: r.floatingIps && JSON.stringify(r.floatingIps),
|
||||
certificates: r.certificates && JSON.stringify(r.certificates),
|
||||
cost: ((c) => c && JSON.stringify(c))(estimateCost(r)),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
@@ -204,6 +293,20 @@ export interface HetznerProblem {
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
/** What else of a project the problems look at: each is null when it could not be read, which says nothing is wrong. */
|
||||
export interface HetznerExtras {
|
||||
loadBalancers?: HetznerLoadBalancer[] | null;
|
||||
floatingIps?: HetznerFloatingIp[] | null;
|
||||
certificates?: HetznerCertificate[] | null;
|
||||
}
|
||||
|
||||
const MONTHS = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"];
|
||||
/** "24 June", in UTC (Hetzner's month is). */
|
||||
export const dayText = (iso: string): string => {
|
||||
const d = new Date(iso);
|
||||
return `${d.getUTCDate()} ${MONTHS[d.getUTCMonth()]}`;
|
||||
};
|
||||
|
||||
/** In the units the pages show (binary, written GB or TB), so an alert and the tile beside it agree. */
|
||||
export const sizeText = (bytes: number): string => {
|
||||
const gb = bytes / 1024 ** 3;
|
||||
@@ -216,7 +319,7 @@ const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / (2
|
||||
/**
|
||||
* Pure: what is wrong with one project, from what is stored for it. `backupStaleHours` is the same limit as for Proxmox guests' backups (0 = not judged).
|
||||
*/
|
||||
export function hetznerProblems(id: number, name: string, servers: StoredServer[], images: HetznerImage[] | null, nowMs: number, backupStaleHours: number): HetznerProblem[] {
|
||||
export function hetznerProblems(id: number, name: string, servers: StoredServer[], images: HetznerImage[] | null, nowMs: number, backupStaleHours: number, extras: HetznerExtras = {}): HetznerProblem[] {
|
||||
const out: HetznerProblem[] = [];
|
||||
for (const s of [...servers].sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
const since = nowMs - Date.parse(s.statusSince);
|
||||
@@ -230,6 +333,15 @@ export function hetznerProblems(id: number, name: string, servers: StoredServer[
|
||||
}
|
||||
if (s.includedTraffic !== null && s.includedTraffic > 0 && s.outgoingTraffic !== null) {
|
||||
const used = s.outgoingTraffic / s.includedTraffic;
|
||||
const forecast = used < TRAFFIC_WARN ? trafficForecast(s, nowMs) : null;
|
||||
if (forecast?.exhaustedAt) {
|
||||
out.push({
|
||||
key: `hetzner-forecast:${id}:${s.id}`,
|
||||
serverId: s.id,
|
||||
severity: "warning",
|
||||
message: `${name}: at this rate ${s.name} will pass its ${sizeText(s.includedTraffic)} traffic quota on ${dayText(forecast.exhaustedAt)} — it has sent ${sizeText(s.outgoingTraffic)} so far, about ${sizeText(forecast.perDay)} a day (${forecast.basis === "recent" ? "the last week's rate" : "the month's average"})`,
|
||||
});
|
||||
}
|
||||
if (used >= TRAFFIC_WARN) {
|
||||
out.push({
|
||||
key: `hetzner-traffic:${id}:${s.id}`,
|
||||
@@ -249,6 +361,32 @@ export function hetznerProblems(id: number, name: string, servers: StoredServer[
|
||||
}
|
||||
}
|
||||
}
|
||||
const serverName = (sid: number | null) => (sid === null ? null : (servers.find((x) => x.id === sid)?.name ?? `#${sid}`));
|
||||
for (const lb of [...(extras.loadBalancers ?? [])].sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
const unhealthy = lb.targets.filter((t) => t.health === "unhealthy");
|
||||
if (unhealthy.length === 0) continue;
|
||||
const names = [...new Set(unhealthy.map((t) => serverName(t.serverId) ?? t.label ?? "a target"))];
|
||||
out.push({
|
||||
key: `hetzner-lb:${id}:${lb.id}`,
|
||||
serverId: 0,
|
||||
severity: unhealthy.length === lb.targets.length ? "critical" : "warning",
|
||||
message: `${name}: load balancer ${lb.name} has ${unhealthy.length} of ${plural(lb.targets.length, "target")} unhealthy (${names.join(", ")})${unhealthy.length === lb.targets.length ? " — it has nothing healthy to send to" : ""}`,
|
||||
});
|
||||
}
|
||||
for (const c of [...(extras.certificates ?? [])].sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
if (c.usedBy === 0) continue; // one nothing uses ending is no news
|
||||
const label = `certificate ${c.name}${c.domains.length > 0 ? ` (${c.domains.slice(0, 3).join(", ")}${c.domains.length > 3 ? ", …" : ""})` : ""}`;
|
||||
const left = c.notValidAfter === null ? null : (Date.parse(c.notValidAfter) - nowMs) / DAY;
|
||||
if (c.issuance === "failed" || c.renewal === "failed") {
|
||||
out.push({ key: `hetzner-cert:${id}:${c.id}`, serverId: 0, severity: left !== null && left <= CERT_CRITICAL_DAYS ? "critical" : "warning", message: `${name}: ${label} could not be ${c.issuance === "failed" ? "issued" : "renewed"} by Hetzner${c.error ? ` — ${c.error}` : ""}${left !== null && left > 0 ? ` (it is valid for ${plural(Math.floor(left), "more day")})` : ""}` });
|
||||
} else if (left !== null && left <= CERT_WARN_DAYS) {
|
||||
out.push({ key: `hetzner-cert:${id}:${c.id}`, serverId: 0, severity: left <= CERT_CRITICAL_DAYS ? "critical" : "warning", message: `${name}: ${label} ${left <= 0 ? `ended ${ago(-left * DAY)} ago` : `ends in ${plural(Math.ceil(left), "day")}`}${c.type === "managed" ? " — Hetzner should have renewed it" : " — it was uploaded, so it has to be renewed by hand"}` });
|
||||
}
|
||||
}
|
||||
for (const f of [...(extras.floatingIps ?? [])].sort((a, b) => a.ip.localeCompare(b.ip))) {
|
||||
if (f.serverId !== null) continue;
|
||||
out.push({ key: `hetzner-fip:${id}:${f.id}`, serverId: 0, severity: "warning", message: `${name}: floating IP ${f.ip}${f.name ? ` (${f.name})` : ""} points at no server — it is billed all the same` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
@@ -39,10 +39,12 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^hetzner-(?:off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
|
||||
m = /^hetzner-(?:off|stuck|rescue|traffic|forecast|backup|lb|cert|fip):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^fragnet-(?:disk|memory|backup):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^(?:sonarr|radarr|prowlarr)-(?:health|queue|disk):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^n8n-(?:failed):(\d+)/.exec(key);
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
* severity, integrations, maintenance, things that expire. It reads what the app already knows — nothing here calls out to a server or an
|
||||
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
@@ -13,7 +13,9 @@ import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync
|
||||
import { isNextcloudFresh, nextcloudSyncStatuses } from "./nextcloudSync.js";
|
||||
import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
|
||||
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
|
||||
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf } from "./hetznerSync.js";
|
||||
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf, loadBalancersOf as hetznerLoadBalancersOf, certificatesOf as hetznerCertificatesOf, trafficForecast as hetznerTrafficForecast } from "./hetznerSync.js";
|
||||
import { ARR_KINDS } from "../integrations/arr/adapter.js";
|
||||
import { arrSyncStatuses, distinctDisks as arrDisks, isArrFresh, stuckItems as arrStuck, storedOf as arrStoredOf } from "./arrSync.js";
|
||||
import { fragnetSyncStatuses, isFragnetFresh, newestBackup as fragnetNewestBackup, serversOf as fragnetServersOf } from "./fragnetSync.js";
|
||||
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
|
||||
import { isN8nFresh, n8nSyncStatuses, workflowsOf as n8nWorkflowsOf } from "./n8nSync.js";
|
||||
@@ -101,6 +103,25 @@ async function addSpeedtestMetrics(out: Exposition, now: Date) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Sonarr, Radarr and Prowlarr, for /metrics: how many health findings it has, its queue and how much of it is stuck, the missing items and the free space of each disk. Only while the reading is recent. */
|
||||
async function addArrMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await arrSyncStatuses();
|
||||
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations).where(inArray(integrations.type, ARR_KINDS))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(arrState)) {
|
||||
const row = rows.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isArrFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const state = arrStoredOf(s);
|
||||
const labels = { app: row.type, name: row.name };
|
||||
out.add("homelab_arr_health_issues", "gauge", "How many warnings or errors Sonarr, Radarr or Prowlarr reports about itself.", state.health.filter((h) => h.type === "warning" || h.type === "error").length, labels);
|
||||
if (state.queue) {
|
||||
out.add("homelab_arr_queue_items", "gauge", "How many downloads are in the queue of Sonarr or Radarr.", state.queue.total, labels);
|
||||
out.add("homelab_arr_queue_stuck_items", "gauge", "How many of the queue's downloads are stuck (judged wrong, failed or blocked for two hours or more).", arrStuck(state.queue, now.getTime()).length, labels);
|
||||
}
|
||||
if (state.missing !== null) out.add("homelab_arr_wanted_missing", "gauge", "How many wanted items Sonarr or Radarr has no file for.", state.missing, labels);
|
||||
for (const d of arrDisks(state.disks)) out.add("homelab_arr_disk_free_bytes", "gauge", "Free space of a disk Sonarr, Radarr or Prowlarr uses, in bytes.", d.free, { ...labels, path: d.path });
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Fragnet game server, for /metrics: whether it is running, its disk and memory against their limits and when it was last backed up. Only while the reading is recent. */
|
||||
async function addFragnetMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await fragnetSyncStatuses();
|
||||
@@ -136,6 +157,16 @@ async function addHetznerMetrics(out: Exposition, now: Date) {
|
||||
if (server.outgoingTraffic !== null) out.add("homelab_hetzner_traffic_outgoing_bytes", "gauge", "Outgoing traffic of a Hetzner Cloud server this month, in bytes.", server.outgoingTraffic, labels);
|
||||
if (server.includedTraffic !== null) out.add("homelab_hetzner_traffic_included_bytes", "gauge", "The outgoing traffic a Hetzner Cloud server includes per month, in bytes.", server.includedTraffic, labels);
|
||||
}
|
||||
for (const server of hetznerServersOf(s)) {
|
||||
const forecast = hetznerTrafficForecast(server, now.getTime());
|
||||
if (forecast) out.add("homelab_hetzner_traffic_forecast_bytes", "gauge", "What a Hetzner Cloud server's outgoing traffic is forecast to come to by the end of the month, in bytes.", forecast.projectedMonth, { project: row.name, server: server.name });
|
||||
}
|
||||
for (const lb of hetznerLoadBalancersOf(s) ?? []) {
|
||||
out.add("homelab_hetzner_load_balancer_unhealthy_targets", "gauge", "How many targets of a Hetzner Cloud load balancer are unhealthy.", lb.targets.filter((t) => t.health === "unhealthy").length, { project: row.name, load_balancer: lb.name });
|
||||
}
|
||||
for (const c of hetznerCertificatesOf(s) ?? []) {
|
||||
if (c.notValidAfter) out.add("homelab_hetzner_certificate_expiry_timestamp_seconds", "gauge", "When a Hetzner Cloud certificate ends, in seconds since 1970.", Date.parse(c.notValidAfter) / 1000, { project: row.name, certificate: c.name });
|
||||
}
|
||||
const cost = hetznerCostOf(s);
|
||||
if (cost) out.add("homelab_hetzner_monthly_cost", "gauge", "Estimated monthly cost of a Hetzner Cloud project at list prices, including VAT, in the project's currency.", cost.total, { project: row.name, currency: cost.currency });
|
||||
}
|
||||
@@ -466,6 +497,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addHetznerMetrics(out, now);
|
||||
await addSpeedtestMetrics(out, now);
|
||||
await addFragnetMetrics(out, now);
|
||||
await addArrMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
Reference in new issue
Block a user