New integration kinds sonarr, radarr and prowlarr (shared adapter and sync, page under Apps -> Media managers): own health findings, stuck downloads, nearly full disks. Hetzner now reads load balancers (target health), floating IPs and certificates, includes them in the cost estimate, and forecasts the month's traffic against the quota. Migrations 0072 and 0073; docs, metrics and tests included. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
504 lines
39 KiB
TypeScript
504 lines
39 KiB
TypeScript
/**
|
|
* What /metrics says, in Prometheus's text format: servers (reporting or not, load, memory, disks, updates, agent version), alerts by
|
|
* severity, integrations, maintenance, things that expire. It reads what the app already knows — nothing here calls out to a server or an
|
|
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
|
|
*/
|
|
import { eq, inArray } from "drizzle-orm";
|
|
import { db } from "../db/client.js";
|
|
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState, arrState } from "../db/schema.js";
|
|
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
|
import { levelStatuses } from "./serviceLevels.js";
|
|
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
|
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
|
|
import { isNextcloudFresh, nextcloudSyncStatuses } from "./nextcloudSync.js";
|
|
import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
|
|
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
|
|
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf, loadBalancersOf as hetznerLoadBalancersOf, certificatesOf as hetznerCertificatesOf, trafficForecast as hetznerTrafficForecast } from "./hetznerSync.js";
|
|
import { ARR_KINDS } from "../integrations/arr/adapter.js";
|
|
import { arrSyncStatuses, distinctDisks as arrDisks, isArrFresh, stuckItems as arrStuck, storedOf as arrStoredOf } from "./arrSync.js";
|
|
import { fragnetSyncStatuses, isFragnetFresh, newestBackup as fragnetNewestBackup, serversOf as fragnetServersOf } from "./fragnetSync.js";
|
|
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
|
|
import { isN8nFresh, n8nSyncStatuses, workflowsOf as n8nWorkflowsOf } from "./n8nSync.js";
|
|
import { componentsOf as paperlessComponentsOf, failedOf as paperlessFailedOf, isPaperlessFresh, paperlessSyncStatuses } from "./paperlessSync.js";
|
|
import { immichSyncStatuses, isImmichFresh, queuesOf as immichQueuesOf } from "./immichSync.js";
|
|
import { authentikSyncStatuses, certificatesOf, daysUntil, isAuthentikFresh, outpostsOf } from "./authentikSync.js";
|
|
import { latestTests } from "./restoreTests.js";
|
|
import { heartbeatState, parseStamp } from "./heartbeats.js";
|
|
import { env } from "../env.js";
|
|
import { getAlerts } from "./alerts.js";
|
|
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
|
|
import { getSourceStatus } from "./integrationHealthMonitor.js";
|
|
import { parsePendingUpdates } from "./pendingUpdates.js";
|
|
import { parseHostHealth, poolIsFine, withoutIgnoredServices } from "./hostHealth.js";
|
|
import { parseThresholds as parseServerThresholds } from "./serverThresholds.js";
|
|
import { parseThresholds } from "./serverThresholds.js";
|
|
import { getCachedBackupCoverage } from "./backupCoverage.js";
|
|
import { lastAttempt as lastSelfBackup, listSnapshots } from "./selfBackup.js";
|
|
import { loadAcks, attentionCounts } from "./alertAcks.js";
|
|
import { listIncidents } from "./incidents.js";
|
|
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
|
import { storageFillingSoon, storageOverview } from "./storageHistory.js";
|
|
import { computeSecretStatus } from "./secretStatus.js";
|
|
import { getSettings } from "./settingsStore.js";
|
|
import { getUptime } from "./uptimeHistory.js";
|
|
import { and, gt, lte } from "drizzle-orm";
|
|
|
|
/** A label value as Prometheus wants it: backslash, double quote and newline escaped. */
|
|
export function escapeLabel(value: string): string {
|
|
return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
|
|
}
|
|
|
|
/** A metric name's HELP text may not contain a raw newline either. */
|
|
const help = (text: string) => text.replace(/\\/g, "\\\\").replace(/\n/g, "\\n");
|
|
|
|
type Labels = Record<string, string>;
|
|
|
|
class Exposition {
|
|
private lines: string[] = [];
|
|
private declared = new Set<string>();
|
|
|
|
/** One sample. The HELP and TYPE lines for a metric are written the first time it appears. */
|
|
add(name: string, type: "gauge" | "counter", description: string, value: number, labels: Labels = {}): void {
|
|
if (!Number.isFinite(value)) return;
|
|
if (!this.declared.has(name)) {
|
|
this.declared.add(name);
|
|
this.lines.push(`# HELP ${name} ${help(description)}`, `# TYPE ${name} ${type}`);
|
|
}
|
|
const parts = Object.entries(labels).map(([k, v]) => `${k}="${escapeLabel(v)}"`);
|
|
this.lines.push(`${name}${parts.length ? `{${parts.join(",")}}` : ""} ${Number.isInteger(value) ? value : Number(value.toFixed(4))}`);
|
|
}
|
|
|
|
toString(): string {
|
|
return `${this.lines.join("\n")}\n`;
|
|
}
|
|
}
|
|
|
|
/** Each Nextcloud, for /metrics: whether it is in maintenance mode, the room left for the data, who was active and the app updates waiting. Only while the reading is recent. */
|
|
async function addNextcloudMetrics(out: Exposition, now: Date) {
|
|
const statuses = await nextcloudSyncStatuses();
|
|
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "nextcloud"))).map((r) => [r.id, r]));
|
|
for (const s of await db.select().from(nextcloudState)) {
|
|
const row = names.get(s.integrationId);
|
|
if (!row || !row.enabled || !isNextcloudFresh(statuses[s.integrationId], now.getTime())) continue;
|
|
const labels = { name: row.name };
|
|
out.add("homelab_nextcloud_maintenance_mode", "gauge", "1 if Nextcloud is in maintenance mode, else 0.", s.maintenance ? 1 : 0, labels);
|
|
if (s.freeBytes !== null) out.add("homelab_nextcloud_free_bytes", "gauge", "Bytes free where Nextcloud keeps its data.", s.freeBytes, labels);
|
|
if (s.active24h !== null) out.add("homelab_nextcloud_active_users_24h", "gauge", "Users active in the last 24 hours.", s.active24h, labels);
|
|
if (s.cronState !== null) out.add("homelab_nextcloud_cron_ok", "gauge", "1 if Nextcloud's own check says its background jobs ran within the last hour, else 0; absent when it could not be asked.", s.cronState === "error" ? 0 : 1, labels);
|
|
if (s.appUpdates !== null) out.add("homelab_nextcloud_app_updates", "gauge", "How many apps have an update waiting.", s.appUpdates, labels);
|
|
}
|
|
}
|
|
|
|
/** Each Speedtest Tracker, for /metrics: its newest completed test — download and upload in bits per second, ping in seconds. Only while the reading is recent. */
|
|
async function addSpeedtestMetrics(out: Exposition, now: Date) {
|
|
const statuses = await speedtestSyncStatuses();
|
|
for (const row of await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "speedtest"), eq(integrations.enabled, true)))) {
|
|
if (!isSpeedtestFresh(statuses[row.id], now.getTime())) continue;
|
|
const latest = (await recentSpeedtestResults(row.id, 20)).find((r) => r.status === "completed");
|
|
if (!latest) continue;
|
|
const labels = { name: row.name };
|
|
if (latest.downloadBps !== null) out.add("homelab_speedtest_download_bits_per_second", "gauge", "Download speed of the newest completed speed test, in bits per second.", latest.downloadBps, labels);
|
|
if (latest.uploadBps !== null) out.add("homelab_speedtest_upload_bits_per_second", "gauge", "Upload speed of the newest completed speed test, in bits per second.", latest.uploadBps, labels);
|
|
if (latest.pingMs !== null) out.add("homelab_speedtest_ping_seconds", "gauge", "Ping of the newest completed speed test, in seconds.", latest.pingMs / 1000, labels);
|
|
}
|
|
}
|
|
|
|
/** Each Sonarr, Radarr and Prowlarr, for /metrics: how many health findings it has, its queue and how much of it is stuck, the missing items and the free space of each disk. Only while the reading is recent. */
|
|
async function addArrMetrics(out: Exposition, now: Date) {
|
|
const statuses = await arrSyncStatuses();
|
|
const rows = new Map((await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations).where(inArray(integrations.type, ARR_KINDS))).map((r) => [r.id, r]));
|
|
for (const s of await db.select().from(arrState)) {
|
|
const row = rows.get(s.integrationId);
|
|
if (!row || !row.enabled || !isArrFresh(statuses[s.integrationId], now.getTime())) continue;
|
|
const state = arrStoredOf(s);
|
|
const labels = { app: row.type, name: row.name };
|
|
out.add("homelab_arr_health_issues", "gauge", "How many warnings or errors Sonarr, Radarr or Prowlarr reports about itself.", state.health.filter((h) => h.type === "warning" || h.type === "error").length, labels);
|
|
if (state.queue) {
|
|
out.add("homelab_arr_queue_items", "gauge", "How many downloads are in the queue of Sonarr or Radarr.", state.queue.total, labels);
|
|
out.add("homelab_arr_queue_stuck_items", "gauge", "How many of the queue's downloads are stuck (judged wrong, failed or blocked for two hours or more).", arrStuck(state.queue, now.getTime()).length, labels);
|
|
}
|
|
if (state.missing !== null) out.add("homelab_arr_wanted_missing", "gauge", "How many wanted items Sonarr or Radarr has no file for.", state.missing, labels);
|
|
for (const d of arrDisks(state.disks)) out.add("homelab_arr_disk_free_bytes", "gauge", "Free space of a disk Sonarr, Radarr or Prowlarr uses, in bytes.", d.free, { ...labels, path: d.path });
|
|
}
|
|
}
|
|
|
|
/** Each Fragnet game server, for /metrics: whether it is running, its disk and memory against their limits and when it was last backed up. Only while the reading is recent. */
|
|
async function addFragnetMetrics(out: Exposition, now: Date) {
|
|
const statuses = await fragnetSyncStatuses();
|
|
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "fragnet"))).map((r) => [r.id, r]));
|
|
for (const s of await db.select().from(fragnetState)) {
|
|
const row = names.get(s.integrationId);
|
|
if (!row || !row.enabled || !isFragnetFresh(statuses[s.integrationId], now.getTime())) continue;
|
|
for (const server of fragnetServersOf(s)) {
|
|
const labels = { account: row.name, server: server.name, game: server.game ?? "" };
|
|
out.add("homelab_fragnet_server_running", "gauge", "1 if the Fragnet game server is running, else 0.", server.state === "running" ? 1 : 0, labels);
|
|
if (server.usage) {
|
|
out.add("homelab_fragnet_disk_used_bytes", "gauge", "Disk a Fragnet game server uses, in bytes.", server.usage.diskBytes, labels);
|
|
out.add("homelab_fragnet_memory_used_bytes", "gauge", "Memory a Fragnet game server uses, in bytes.", server.usage.memoryBytes, labels);
|
|
}
|
|
if (server.diskLimitMb) out.add("homelab_fragnet_disk_limit_bytes", "gauge", "The disk a Fragnet game server may use, in bytes.", server.diskLimitMb * 1024 ** 2, labels);
|
|
if (server.memoryLimitMb) out.add("homelab_fragnet_memory_limit_bytes", "gauge", "The memory a Fragnet game server may use, in bytes.", server.memoryLimitMb * 1024 ** 2, labels);
|
|
const backedUp = fragnetNewestBackup(server)?.at;
|
|
if (backedUp) out.add("homelab_fragnet_newest_backup_timestamp_seconds", "gauge", "When the newest successful backup of a Fragnet game server finished, in seconds since 1970.", Date.parse(backedUp) / 1000, labels);
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Each Hetzner Cloud server, for /metrics: whether it is running, its traffic this month against the quota, and the project's monthly cost estimate. Only while the reading is recent. */
|
|
async function addHetznerMetrics(out: Exposition, now: Date) {
|
|
const statuses = await hetznerSyncStatuses();
|
|
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "hetzner"))).map((r) => [r.id, r]));
|
|
for (const s of await db.select().from(hetznerState)) {
|
|
const row = names.get(s.integrationId);
|
|
if (!row || !row.enabled || !isHetznerFresh(statuses[s.integrationId], now.getTime())) continue;
|
|
for (const server of hetznerServersOf(s)) {
|
|
const labels = { project: row.name, server: server.name };
|
|
out.add("homelab_hetzner_server_running", "gauge", "1 if the Hetzner Cloud server is running, else 0.", server.status === "running" ? 1 : 0, labels);
|
|
if (server.outgoingTraffic !== null) out.add("homelab_hetzner_traffic_outgoing_bytes", "gauge", "Outgoing traffic of a Hetzner Cloud server this month, in bytes.", server.outgoingTraffic, labels);
|
|
if (server.includedTraffic !== null) out.add("homelab_hetzner_traffic_included_bytes", "gauge", "The outgoing traffic a Hetzner Cloud server includes per month, in bytes.", server.includedTraffic, labels);
|
|
}
|
|
for (const server of hetznerServersOf(s)) {
|
|
const forecast = hetznerTrafficForecast(server, now.getTime());
|
|
if (forecast) out.add("homelab_hetzner_traffic_forecast_bytes", "gauge", "What a Hetzner Cloud server's outgoing traffic is forecast to come to by the end of the month, in bytes.", forecast.projectedMonth, { project: row.name, server: server.name });
|
|
}
|
|
for (const lb of hetznerLoadBalancersOf(s) ?? []) {
|
|
out.add("homelab_hetzner_load_balancer_unhealthy_targets", "gauge", "How many targets of a Hetzner Cloud load balancer are unhealthy.", lb.targets.filter((t) => t.health === "unhealthy").length, { project: row.name, load_balancer: lb.name });
|
|
}
|
|
for (const c of hetznerCertificatesOf(s) ?? []) {
|
|
if (c.notValidAfter) out.add("homelab_hetzner_certificate_expiry_timestamp_seconds", "gauge", "When a Hetzner Cloud certificate ends, in seconds since 1970.", Date.parse(c.notValidAfter) / 1000, { project: row.name, certificate: c.name });
|
|
}
|
|
const cost = hetznerCostOf(s);
|
|
if (cost) out.add("homelab_hetzner_monthly_cost", "gauge", "Estimated monthly cost of a Hetzner Cloud project at list prices, including VAT, in the project's currency.", cost.total, { project: row.name, currency: cost.currency });
|
|
}
|
|
}
|
|
|
|
/** Each Docker host's vulnerability counts from Dockhand's scans, for /metrics. Only while the reading is recent, and only for hosts whose summary could be read. */
|
|
async function addDockerVulnMetrics(out: Exposition, now: Date) {
|
|
const statuses = await dockerVulnSyncStatuses();
|
|
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "dockhand"))).map((r) => [r.id, r]));
|
|
for (const s of await db.select().from(dockhandVulnerabilities)) {
|
|
const row = names.get(s.integrationId);
|
|
if (!row || !row.enabled || s.critical === null || !isDockerVulnFresh(statuses[s.integrationId], now.getTime())) continue;
|
|
const labels = { integration: row.name, host: s.environmentName };
|
|
out.add("homelab_docker_vulnerabilities", "gauge", "Vulnerabilities in the scanned images of a Docker host, by severity, from Dockhand's stored scans.", s.critical, { ...labels, severity: "critical" });
|
|
if (s.high !== null) out.add("homelab_docker_vulnerabilities", "gauge", "Vulnerabilities in the scanned images of a Docker host, by severity, from Dockhand's stored scans.", s.high, { ...labels, severity: "high" });
|
|
if (s.medium !== null) out.add("homelab_docker_vulnerabilities", "gauge", "Vulnerabilities in the scanned images of a Docker host, by severity, from Dockhand's stored scans.", s.medium, { ...labels, severity: "medium" });
|
|
if (s.low !== null) out.add("homelab_docker_vulnerabilities", "gauge", "Vulnerabilities in the scanned images of a Docker host, by severity, from Dockhand's stored scans.", s.low, { ...labels, severity: "low" });
|
|
if (s.imagesScanned !== null) out.add("homelab_docker_images_scanned", "gauge", "How many of a Docker host's images have a vulnerability scan in Dockhand.", s.imagesScanned, labels);
|
|
}
|
|
}
|
|
|
|
/** Each NetBox, for /metrics: the workers running and what it holds. Only while the reading is recent. */
|
|
async function addNetBoxMetrics(out: Exposition, now: Date) {
|
|
const statuses = await netboxSyncStatuses();
|
|
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "netbox"))).map((r) => [r.id, r]));
|
|
for (const s of await db.select().from(netboxState)) {
|
|
const row = names.get(s.integrationId);
|
|
if (!row || !row.enabled || !isNetBoxFresh(statuses[s.integrationId], now.getTime())) continue;
|
|
const labels = { name: row.name };
|
|
if (s.workersRunning !== null) out.add("homelab_netbox_workers", "gauge", "How many background workers NetBox has running.", s.workersRunning, labels);
|
|
if (s.devices !== null) out.add("homelab_netbox_devices", "gauge", "How many devices NetBox holds.", s.devices, labels);
|
|
if (s.ipAddresses !== null) out.add("homelab_netbox_ip_addresses", "gauge", "How many IP addresses NetBox holds.", s.ipAddresses, labels);
|
|
}
|
|
}
|
|
|
|
/** Each n8n, for /metrics: how many workflows there are, are switched on and have failed on their last run. Only while the reading is recent. */
|
|
async function addN8nMetrics(out: Exposition, now: Date) {
|
|
const statuses = await n8nSyncStatuses();
|
|
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "n8n"))).map((r) => [r.id, r]));
|
|
for (const s of await db.select().from(n8nState)) {
|
|
const row = names.get(s.integrationId);
|
|
if (!row || !row.enabled || !isN8nFresh(statuses[s.integrationId], now.getTime())) continue;
|
|
const labels = { name: row.name };
|
|
const workflows = n8nWorkflowsOf(s);
|
|
out.add("homelab_n8n_workflows", "gauge", "How many workflows n8n has.", workflows.length, labels);
|
|
out.add("homelab_n8n_workflows_active", "gauge", "How many of them are switched on.", workflows.filter((w) => w.active).length, labels);
|
|
if (s.executionsKnown) out.add("homelab_n8n_workflows_failing", "gauge", "Active workflows whose last run failed.", workflows.filter((w) => w.active && (w.lastStatus === "error" || w.lastStatus === "crashed")).length, labels);
|
|
}
|
|
}
|
|
|
|
/** Each Paperless-ngx, for /metrics: the documents, the inbox, the imports that failed and whether each component of its status is in order. Only while the reading is recent. */
|
|
async function addPaperlessMetrics(out: Exposition, now: Date) {
|
|
const statuses = await paperlessSyncStatuses();
|
|
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "paperless"))).map((r) => [r.id, r]));
|
|
for (const s of await db.select().from(paperlessState)) {
|
|
const row = names.get(s.integrationId);
|
|
if (!row || !row.enabled || !isPaperlessFresh(statuses[s.integrationId], now.getTime())) continue;
|
|
const labels = { name: row.name };
|
|
if (s.documents !== null) out.add("homelab_paperless_documents", "gauge", "How many documents Paperless holds.", s.documents, labels);
|
|
if (s.inbox !== null) out.add("homelab_paperless_inbox", "gauge", "How many documents are waiting in Paperless's inbox.", s.inbox, labels);
|
|
const failed = paperlessFailedOf(s);
|
|
if (failed) out.add("homelab_paperless_failed_imports", "gauge", "Imports that failed in the last week and were not dismissed.", failed.length, labels);
|
|
for (const c of paperlessComponentsOf(s) ?? []) out.add("homelab_paperless_component_ok", "gauge", "1 if the part of Paperless named is in order, else 0.", c.status === "OK" ? 1 : 0, { name: row.name, component: c.name });
|
|
}
|
|
}
|
|
|
|
/** Each Immich, for /metrics: the photos and videos, what they take, how full the disk is and the jobs that failed. Only while the reading is recent. */
|
|
async function addImmichMetrics(out: Exposition, now: Date) {
|
|
const statuses = await immichSyncStatuses();
|
|
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "immich"))).map((r) => [r.id, r]));
|
|
for (const s of await db.select().from(immichState)) {
|
|
const row = names.get(s.integrationId);
|
|
if (!row || !row.enabled || !isImmichFresh(statuses[s.integrationId], now.getTime())) continue;
|
|
const labels = { name: row.name };
|
|
if (s.photos !== null) out.add("homelab_immich_photos", "gauge", "How many photos Immich holds.", s.photos, labels);
|
|
if (s.videos !== null) out.add("homelab_immich_videos", "gauge", "How many videos Immich holds.", s.videos, labels);
|
|
if (s.usageBytes !== null) out.add("homelab_immich_library_bytes", "gauge", "What Immich's library takes, in bytes.", s.usageBytes, labels);
|
|
if (s.diskPercent !== null) out.add("homelab_immich_disk_used_percent", "gauge", "How full the disk under Immich's library is.", s.diskPercent, labels);
|
|
const queues = immichQueuesOf(s);
|
|
if (queues) out.add("homelab_immich_failed_jobs", "gauge", "Background jobs that have failed, all queues together.", queues.reduce((n, q) => n + q.failed, 0), labels);
|
|
}
|
|
}
|
|
|
|
/** Each Authentik, for /metrics: whether each outpost that serves something is reporting, how many workers there are, the failed sign-ins and the days left of each certificate. Only while the reading is recent. */
|
|
async function addAuthentikMetrics(out: Exposition, now: Date) {
|
|
const statuses = await authentikSyncStatuses();
|
|
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "authentik"))).map((r) => [r.id, r]));
|
|
for (const s of await db.select().from(authentikState)) {
|
|
const row = names.get(s.integrationId);
|
|
if (!row || !row.enabled || !isAuthentikFresh(statuses[s.integrationId], now.getTime())) continue;
|
|
for (const o of outpostsOf(s)) out.add("homelab_authentik_outpost_up", "gauge", "1 if the Authentik outpost has reported in lately, else 0.", o.healthy > 0 ? 1 : 0, { name: row.name, outpost: o.name });
|
|
if (s.workers !== null) out.add("homelab_authentik_workers", "gauge", "How many Authentik workers are connected.", s.workers, { name: row.name });
|
|
if (s.failedLogins24h !== null) out.add("homelab_authentik_failed_logins_24h", "gauge", "Sign-ins that failed in the last day (100 means at least that many).", s.failedLogins24h, { name: row.name });
|
|
for (const c of certificatesOf(s)) out.add("homelab_authentik_certificate_expiry_days", "gauge", "Whole days until the certificate runs out; negative once it has.", daysUntil(c.expires, now), { name: row.name, certificate: c.name });
|
|
}
|
|
}
|
|
|
|
/** Each Jellyfin, for /metrics: how many are streaming, how many of those are transcoded, and whether it wants a restart. Only while the reading is recent. */
|
|
async function addJellyfinMetrics(out: Exposition, now: Date) {
|
|
const statuses = await jellyfinSyncStatuses();
|
|
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "jellyfin"))).map((r) => [r.id, r]));
|
|
for (const s of await db.select().from(jellyfinState)) {
|
|
const row = names.get(s.integrationId);
|
|
if (!row || !row.enabled || !isJellyfinFresh(statuses[s.integrationId], now.getTime())) continue;
|
|
const labels = { name: row.name };
|
|
const streams = streamsOf(s);
|
|
out.add("homelab_jellyfin_streams", "gauge", "How many are streaming from Jellyfin right now.", streams.length, labels);
|
|
out.add("homelab_jellyfin_transcodes", "gauge", "How many of those streams Jellyfin is transcoding.", streams.filter((x) => x.transcoding).length, labels);
|
|
out.add("homelab_jellyfin_pending_restart", "gauge", "1 if Jellyfin is waiting to be restarted, else 0.", s.pendingRestart ? 1 : 0, labels);
|
|
}
|
|
}
|
|
|
|
/** Each Duplicati backup, for /metrics: whether it is in order, how long since it last succeeded, and how big what it backs up is. */
|
|
async function addDuplicatiMetrics(out: Exposition, now: Date) {
|
|
const { sources, backups } = await listDuplicati();
|
|
const names = new Map(sources.map((s) => [s.id, s.name]));
|
|
for (const b of backups) {
|
|
const labels = { backup: b.name, source: names.get(b.sourceId) ?? "" };
|
|
const state = backupState(b, now.getTime());
|
|
out.add("homelab_duplicati_backup_ok", "gauge", "1 if the Duplicati backup's last run worked and it is not overdue, else 0.", state === "ok" || state === "warning" ? 1 : 0, labels);
|
|
if (b.lastSuccessAt) out.add("homelab_duplicati_last_success_age_seconds", "gauge", "Seconds since the Duplicati backup last succeeded (a run with warnings counts).", Math.max(0, Math.round((now.getTime() - Date.parse(b.lastSuccessAt)) / 1000)), labels);
|
|
if (b.sizeBytes !== null) out.add("homelab_duplicati_source_size_bytes", "gauge", "The size of what the Duplicati backup examined, as its last report said.", b.sizeBytes, labels);
|
|
}
|
|
}
|
|
|
|
/** Each service-level target, for /metrics: its target, how it is doing this month and how much error budget is left. */
|
|
async function addServiceLevelMetrics(out: Exposition, now: Date) {
|
|
for (const s of await levelStatuses(now, { history: false })) {
|
|
const labels = { name: s.name, subject: s.subjectName ?? "", type: s.subjectType };
|
|
out.add("homelab_service_level_target_percent", "gauge", "The availability a service-level target asks for.", s.target, labels);
|
|
if (s.measure.percent === null) continue;
|
|
out.add("homelab_service_level_availability_percent", "gauge", "How available the thing has been so far this calendar month.", s.measure.percent, labels);
|
|
out.add("homelab_service_level_error_budget_remaining_minutes", "gauge", "Downtime still allowed this month before the target is missed (negative once it is).", Math.round(s.budget.remainingMinutes * 10) / 10, labels);
|
|
}
|
|
}
|
|
|
|
/** Each UPS, for /metrics: whether it is on battery, how full the battery is, how long it would last and how loaded it is. */
|
|
async function addUpsMetrics(out: Exposition, now: Date) {
|
|
const fresh = new Set<number>();
|
|
const nowMs = now.getTime();
|
|
const statuses = await upsSyncStatuses();
|
|
for (const [id, s] of Object.entries(statuses)) if (isUpsFresh(s, nowMs)) fresh.add(Number(id));
|
|
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "ups"))).map((r) => [r.id, r]));
|
|
for (const s of await db.select().from(upsState)) {
|
|
const row = names.get(s.integrationId);
|
|
if (!row || !row.enabled || !fresh.has(s.integrationId)) continue;
|
|
const labels = { name: row.name };
|
|
out.add("homelab_ups_on_battery", "gauge", "1 if the UPS has lost mains power and runs on its battery, else 0.", s.onBattery ? 1 : 0, labels);
|
|
if (s.charge !== null) out.add("homelab_ups_battery_charge_percent", "gauge", "How full the UPS's battery is.", s.charge, labels);
|
|
if (s.runtimeSeconds !== null) out.add("homelab_ups_runtime_seconds", "gauge", "How long the UPS's battery would last at the present load.", s.runtimeSeconds, labels);
|
|
if (s.loadPercent !== null) out.add("homelab_ups_load_percent", "gauge", "How much of its capacity the UPS's load uses.", s.loadPercent, labels);
|
|
if (s.inputVoltage !== null) out.add("homelab_ups_input_volts", "gauge", "The mains voltage the UPS sees.", s.inputVoltage, labels);
|
|
}
|
|
}
|
|
|
|
/** Endpoint checks and restore tests, for /metrics. */
|
|
async function addEndpointMetrics(out: Exposition, now: Date) {
|
|
for (const c of await db.select().from(endpointChecks)) {
|
|
if (!c.enabled || c.lastOk === null) continue;
|
|
const labels = { name: c.name, kind: c.kind };
|
|
out.add("homelab_endpoint_up", "gauge", "1 if the endpoint check's last answer was fine, else 0.", c.lastOk ? 1 : 0, labels);
|
|
if (c.valuePath && c.lastValue !== null && c.lastValue.trim() !== "" && Number.isFinite(Number(c.lastValue))) out.add("homelab_endpoint_value", "gauge", "The number a value check last read from the endpoint's answer.", Number(c.lastValue), labels);
|
|
if (c.lastResponseMs !== null) out.add("homelab_endpoint_response_milliseconds", "gauge", "How long the endpoint took to answer the last time.", c.lastResponseMs, labels);
|
|
if (c.certExpiresAt) out.add("homelab_endpoint_certificate_days_left", "gauge", "Days until the certificate the endpoint shows expires (negative once it has).", Math.floor((Date.parse(c.certExpiresAt) - now.getTime()) / 86_400_000), labels);
|
|
}
|
|
for (const r of (await latestTests()).values()) {
|
|
if (r.result === "not_needed") continue;
|
|
out.add("homelab_restore_test_age_days", "gauge", "Days since a restore of this was last tested.", Math.max(0, Math.floor((now.getTime() - Date.parse(`${r.testedAt}T12:00:00Z`)) / 86_400_000)), { subject: r.label });
|
|
out.add("homelab_restore_test_passed", "gauge", "1 if the last restore test passed (fully or partly), 0 if it failed.", r.result === "failed" ? 0 : 1, { subject: r.label });
|
|
}
|
|
}
|
|
|
|
/** Jobs that report in, for /metrics. */
|
|
async function addHeartbeatMetrics(out: Exposition, now: Date) {
|
|
for (const h of await db.select().from(heartbeats)) {
|
|
if (!h.enabled) continue;
|
|
const labels = { name: h.name };
|
|
const state = heartbeatState(h, now);
|
|
out.add("homelab_heartbeat_up", "gauge", "1 if the job has reported in time and not said it failed, else 0.", state === "late" || state === "failed" ? 0 : 1, labels);
|
|
if (h.lastPingAt) out.add("homelab_heartbeat_last_ping_timestamp_seconds", "gauge", "When the job last reported that it finished (Unix time).", Math.floor(parseStamp(h.lastPingAt) / 1000), labels);
|
|
if (h.lastDurationMs !== null) out.add("homelab_heartbeat_last_duration_seconds", "gauge", "How long the job said its last run took.", Math.round(h.lastDurationMs / 100) / 10, labels);
|
|
}
|
|
}
|
|
|
|
export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
|
const started = Date.now();
|
|
const out = new Exposition();
|
|
const settings = await getSettings();
|
|
const nowSec = Math.floor(now.getTime() / 1000);
|
|
|
|
out.add("homelab_info", "gauge", "Always 1; the labels say which installation this is.", 1, { environment: env.environment?.name || "" });
|
|
|
|
// ── Servers ──
|
|
const latest = latestAgentVersions();
|
|
const serverRows = await db.select().from(servers);
|
|
for (const s of serverRows) {
|
|
// A server may have its own "offline after"; otherwise the global one.
|
|
const offlineAfterMs = (parseThresholds(s.thresholds)?.serverOfflineMinutes ?? settings.healthChecks.serverOfflineMinutes) * 60_000;
|
|
if (!s.lastSeenAt) continue; // never reported: nothing to say about it
|
|
const server = s.name;
|
|
const seen = Date.parse(s.lastSeenAt);
|
|
out.add("homelab_server_up", "gauge", "1 if the server's agent has reported within the offline threshold, else 0.", now.getTime() - seen <= offlineAfterMs ? 1 : 0, { server });
|
|
out.add("homelab_server_last_seen_timestamp_seconds", "gauge", "When the server's agent last reported, as a Unix time.", Math.floor(seen / 1000), { server });
|
|
if (s.cpuLoadPercent !== null) out.add("homelab_server_cpu_load_percent", "gauge", "CPU load as the agent reports it.", s.cpuLoadPercent, { server });
|
|
if (s.memTotalBytes && s.memUsedBytes !== null) out.add("homelab_server_memory_used_percent", "gauge", "Memory in use.", (s.memUsedBytes / s.memTotalBytes) * 100, { server });
|
|
let disks: { mount: string; sizeBytes: number; usedBytes: number }[] = [];
|
|
try {
|
|
disks = s.disks ? JSON.parse(s.disks) : [];
|
|
} catch {
|
|
// a garbled report shouldn't blank the whole scrape
|
|
}
|
|
for (const d of disks) {
|
|
if (d.sizeBytes > 0) out.add("homelab_server_disk_used_percent", "gauge", "How full a disk is.", (d.usedBytes / d.sizeBytes) * 100, { server, mount: d.mount });
|
|
}
|
|
const updates = parsePendingUpdates(s.pendingUpdates);
|
|
if (updates) {
|
|
out.add("homelab_server_pending_updates", "gauge", "Updates the agent says are waiting to be installed.", updates.count, { server });
|
|
if (updates.security !== null) out.add("homelab_server_pending_security_updates", "gauge", "Of those, the ones that are security fixes.", updates.security, { server });
|
|
out.add("homelab_server_reboot_required", "gauge", "1 if a restart is due.", updates.rebootRequired ? 1 : 0, { server });
|
|
}
|
|
const health = withoutIgnoredServices(parseHostHealth(s.hostHealth), parseServerThresholds(s.thresholds)?.ignoreServices);
|
|
if (health?.failedUnits) out.add("homelab_server_failed_units", "gauge", "Services (systemd units) that have failed on the server.", health.failedUnits.length, { server });
|
|
for (const d of health?.smart ?? []) {
|
|
if (d.healthy !== null) out.add("homelab_server_disk_healthy", "gauge", "1 if the disk's own SMART self-assessment passes, 0 if it fails.", d.healthy ? 1 : 0, { server, device: d.device });
|
|
if (d.wearPercent !== null) out.add("homelab_server_disk_wear_percent", "gauge", "How much of an SSD's rated life is used.", d.wearPercent, { server, device: d.device });
|
|
if (d.temperatureC !== null) out.add("homelab_server_disk_temperature_celsius", "gauge", "A disk's temperature.", d.temperatureC, { server, device: d.device });
|
|
}
|
|
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a storage pool (ZFS, or Storage Spaces on Windows) is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
|
|
out.add("homelab_server_agent_outdated", "gauge", "1 if the server's agent is older than the one this installation would install now.", isAgentOutdated(s, latest) ? 1 : 0, { server });
|
|
}
|
|
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
|
|
try {
|
|
for (const s of serverRows) {
|
|
for (const f of disksFillingSoon(await getForecastSeries(s.id, now), 3650, 101)) {
|
|
out.add("homelab_server_disk_days_to_full", "gauge", "Days until a disk is full at its present rate of filling; only for disks with a steady climb.", f.daysToFull, { server: s.name, mount: f.mount });
|
|
}
|
|
}
|
|
} catch (err) {
|
|
console.error("[metrics] couldn't work out the disk forecasts:", err);
|
|
}
|
|
try {
|
|
for (const f of storageFillingSoon(await storageOverview(now), 3650, 101)) {
|
|
out.add("homelab_storage_days_to_full", "gauge", "Days until a Proxmox storage or Synology volume is full at its present rate of filling; only for those with a steady climb.", f.daysToFull, { pool: f.label });
|
|
}
|
|
} catch (err) {
|
|
console.error("[metrics] couldn't work out the storage forecasts:", err);
|
|
}
|
|
try {
|
|
const uptime = await getUptime(now, 30);
|
|
for (const s of serverRows) {
|
|
const series = uptime.servers[s.id];
|
|
if (series) out.add("homelab_server_uptime_ratio_30d", "gauge", "Share of the last 30 days a server was up, 0 to 1.", series.percent / 100, { server: s.name });
|
|
}
|
|
} catch (err) {
|
|
console.error("[metrics] couldn't read the uptime:", err);
|
|
}
|
|
|
|
// ── Homelab Manager's own backups ──
|
|
try {
|
|
const snapshots = listSnapshots();
|
|
out.add("homelab_selfbackup_snapshots", "gauge", "How many backups of Homelab Manager's own database are kept.", snapshots.length);
|
|
const newest = snapshots.find((s) => s.kind === "scheduled" || s.kind === "manual");
|
|
if (newest) out.add("homelab_selfbackup_last_success_timestamp_seconds", "gauge", "When the newest backup of Homelab Manager's own database was made, as a Unix time.", Math.floor(Date.parse(newest.at) / 1000));
|
|
const last = await lastSelfBackup();
|
|
if (last) out.add("homelab_selfbackup_last_attempt_ok", "gauge", "1 if the last attempt to back up Homelab Manager's own database worked, 0 if it failed.", last.ok ? 1 : 0);
|
|
} catch (err) {
|
|
console.error("[metrics] couldn't read the backups folder:", err);
|
|
}
|
|
|
|
// ── Alerts ──
|
|
try {
|
|
const report = await getAlerts(false);
|
|
// What the backup check last found (the alert pass above refreshes it); never reads Proxmox or Backup Server itself.
|
|
const coverage = await getCachedBackupCoverage(now);
|
|
if (coverage) {
|
|
for (const status of ["ok", "stale", "failed", "none", "unknown"] as const) out.add("homelab_backup_guests", "gauge", "Proxmox guests by how well they are backed up (ok, stale, failed, none, unknown).", coverage.counts[status], { status });
|
|
for (const g of coverage.guests) {
|
|
if (g.lastBackupAt) out.add("homelab_backup_last_age_seconds", "gauge", "How long ago a guest's newest backup was.", Math.max(0, Math.floor((now.getTime() - Date.parse(g.lastBackupAt)) / 1000)), { guest: g.name, vmid: String(g.vmid), integration: g.integrationName });
|
|
}
|
|
}
|
|
for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity });
|
|
// Somebody has said they are dealing with these (still raised, and still counted above).
|
|
out.add("homelab_alerts_acknowledged", "gauge", "Alerts somebody has acknowledged and that are not silenced by maintenance.", attentionCounts(report.alerts, await loadAcks(now)).acknowledged);
|
|
out.add("homelab_incidents_open", "gauge", "Incidents that have begun and not yet ended.", (await listIncidents({ limit: 1, now })).filter((i) => i.open).length);
|
|
out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced);
|
|
out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped);
|
|
} catch (err) {
|
|
console.error("[metrics] couldn't read the alerts:", err);
|
|
}
|
|
|
|
// ── Integrations ──
|
|
for (const i of await db.select({ name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations)) {
|
|
if (!i.enabled) continue;
|
|
const status = getSourceStatus(i.type).status;
|
|
if (status === "unknown") continue; // nothing has called it yet: not up, not down
|
|
out.add("homelab_integration_up", "gauge", "1 if the integration's latest calls are working, 0 if they are failing.", status === "ok" ? 1 : 0, { integration: i.name, type: i.type });
|
|
}
|
|
|
|
// ── Maintenance ──
|
|
const iso = now.toISOString();
|
|
const active = await db.select({ id: maintenanceWindows.id }).from(maintenanceWindows).where(and(lte(maintenanceWindows.startedAt, iso), gt(maintenanceWindows.endsAt, iso)));
|
|
out.add("homelab_maintenance_active_windows", "gauge", "Maintenance windows running now.", active.length);
|
|
|
|
// ── Things that expire ──
|
|
for (const s of await db.select().from(secrets)) {
|
|
const status = computeSecretStatus(s.expiryDate, s.warnDays, now);
|
|
if (status.daysLeft !== null) out.add("homelab_secret_expiry_days", "gauge", "Days until a secret expires (negative once it has).", status.daysLeft, { name: s.name });
|
|
}
|
|
|
|
// ── Inventory ──
|
|
const power = (await db.select({ w: assets.powerWatts }).from(assets)).reduce((n, a) => n + (a.w ?? 0), 0);
|
|
out.add("homelab_inventory_power_watts", "gauge", "Typical power draw of everything in the inventory, in watts.", power);
|
|
|
|
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
|
|
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
|
|
await addEndpointMetrics(out, now);
|
|
await addUpsMetrics(out, now);
|
|
await addServiceLevelMetrics(out, now);
|
|
await addDuplicatiMetrics(out, now);
|
|
await addJellyfinMetrics(out, now);
|
|
await addAuthentikMetrics(out, now);
|
|
await addImmichMetrics(out, now);
|
|
await addPaperlessMetrics(out, now);
|
|
await addN8nMetrics(out, now);
|
|
await addNetBoxMetrics(out, now);
|
|
await addNextcloudMetrics(out, now);
|
|
await addDockerVulnMetrics(out, now);
|
|
await addHetznerMetrics(out, now);
|
|
await addSpeedtestMetrics(out, now);
|
|
await addFragnetMetrics(out, now);
|
|
await addArrMetrics(out, now);
|
|
await addHeartbeatMetrics(out, now);
|
|
return out.toString();
|
|
}
|