Add runbooks, backup coverage, inventory, roles from groups, Prometheus metrics and event webhooks
- Runbooks: notes per server/integration, shown next to matching alerts - Backup coverage page across Proxmox and PBS, with a digest section - Hardware inventory with warranty, power and running cost - Authentik group to role mapping with preview and safeguards - /metrics (bearer token) and signed outgoing event webhooks - Docs: README, DATABASE, NOTIFICATIONS, ROLES and new API.md Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c36e2092c5
commit
c17e9aa140
55 files changed
+7177
-27
No files matched your search
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* What /metrics says, in Prometheus's text format: servers (reporting or not, load, memory, disks, updates, agent version), alerts by
|
||||
* severity, integrations, maintenance, things that expire. It reads what the app already knows — nothing here calls out to a server or an
|
||||
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
|
||||
import { env } from "../env.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
|
||||
import { getSourceStatus } from "./integrationHealthMonitor.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
import { and, gt, lte } from "drizzle-orm";
|
||||
|
||||
/** A label value as Prometheus wants it: backslash, double quote and newline escaped. */
|
||||
export function escapeLabel(value: string): string {
|
||||
return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
|
||||
}
|
||||
|
||||
/** A metric name's HELP text may not contain a raw newline either. */
|
||||
const help = (text: string) => text.replace(/\\/g, "\\\\").replace(/\n/g, "\\n");
|
||||
|
||||
type Labels = Record<string, string>;
|
||||
|
||||
class Exposition {
|
||||
private lines: string[] = [];
|
||||
private declared = new Set<string>();
|
||||
|
||||
/** One sample. The HELP and TYPE lines for a metric are written the first time it appears. */
|
||||
add(name: string, type: "gauge" | "counter", description: string, value: number, labels: Labels = {}): void {
|
||||
if (!Number.isFinite(value)) return;
|
||||
if (!this.declared.has(name)) {
|
||||
this.declared.add(name);
|
||||
this.lines.push(`# HELP ${name} ${help(description)}`, `# TYPE ${name} ${type}`);
|
||||
}
|
||||
const parts = Object.entries(labels).map(([k, v]) => `${k}="${escapeLabel(v)}"`);
|
||||
this.lines.push(`${name}${parts.length ? `{${parts.join(",")}}` : ""} ${Number.isInteger(value) ? value : Number(value.toFixed(4))}`);
|
||||
}
|
||||
|
||||
toString(): string {
|
||||
return `${this.lines.join("\n")}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
const started = Date.now();
|
||||
const out = new Exposition();
|
||||
const settings = await getSettings();
|
||||
const nowSec = Math.floor(now.getTime() / 1000);
|
||||
|
||||
out.add("homelab_info", "gauge", "Always 1; the labels say which installation this is.", 1, { environment: env.environment?.name || "" });
|
||||
|
||||
// ── Servers ──
|
||||
const latest = latestAgentVersions();
|
||||
const serverRows = await db.select().from(servers);
|
||||
const offlineAfterMs = settings.healthChecks.serverOfflineMinutes * 60_000;
|
||||
for (const s of serverRows) {
|
||||
if (!s.lastSeenAt) continue; // never reported: nothing to say about it
|
||||
const server = s.name;
|
||||
const seen = Date.parse(s.lastSeenAt);
|
||||
out.add("homelab_server_up", "gauge", "1 if the server's agent has reported within the offline threshold, else 0.", now.getTime() - seen <= offlineAfterMs ? 1 : 0, { server });
|
||||
out.add("homelab_server_last_seen_timestamp_seconds", "gauge", "When the server's agent last reported, as a Unix time.", Math.floor(seen / 1000), { server });
|
||||
if (s.cpuLoadPercent !== null) out.add("homelab_server_cpu_load_percent", "gauge", "CPU load as the agent reports it.", s.cpuLoadPercent, { server });
|
||||
if (s.memTotalBytes && s.memUsedBytes !== null) out.add("homelab_server_memory_used_percent", "gauge", "Memory in use.", (s.memUsedBytes / s.memTotalBytes) * 100, { server });
|
||||
let disks: { mount: string; sizeBytes: number; usedBytes: number }[] = [];
|
||||
try {
|
||||
disks = s.disks ? JSON.parse(s.disks) : [];
|
||||
} catch {
|
||||
// a garbled report shouldn't blank the whole scrape
|
||||
}
|
||||
for (const d of disks) {
|
||||
if (d.sizeBytes > 0) out.add("homelab_server_disk_used_percent", "gauge", "How full a disk is.", (d.usedBytes / d.sizeBytes) * 100, { server, mount: d.mount });
|
||||
}
|
||||
const updates = parsePendingUpdates(s.pendingUpdates);
|
||||
if (updates) {
|
||||
out.add("homelab_server_pending_updates", "gauge", "Updates the agent says are waiting to be installed.", updates.count, { server });
|
||||
if (updates.security !== null) out.add("homelab_server_pending_security_updates", "gauge", "Of those, the ones that are security fixes.", updates.security, { server });
|
||||
out.add("homelab_server_reboot_required", "gauge", "1 if a restart is due.", updates.rebootRequired ? 1 : 0, { server });
|
||||
}
|
||||
out.add("homelab_server_agent_outdated", "gauge", "1 if the server's agent is older than the one this installation would install now.", isAgentOutdated(s, latest) ? 1 : 0, { server });
|
||||
}
|
||||
try {
|
||||
const uptime = await getUptime(now, 30);
|
||||
for (const s of serverRows) {
|
||||
const series = uptime.servers[s.id];
|
||||
if (series) out.add("homelab_server_uptime_ratio_30d", "gauge", "Share of the last 30 days a server was up, 0 to 1.", series.percent / 100, { server: s.name });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[metrics] couldn't read the uptime:", err);
|
||||
}
|
||||
|
||||
// ── Alerts ──
|
||||
try {
|
||||
const report = await getAlerts(false);
|
||||
for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity });
|
||||
out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced);
|
||||
out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped);
|
||||
} catch (err) {
|
||||
console.error("[metrics] couldn't read the alerts:", err);
|
||||
}
|
||||
|
||||
// ── Integrations ──
|
||||
for (const i of await db.select({ name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations)) {
|
||||
if (!i.enabled) continue;
|
||||
const status = getSourceStatus(i.type).status;
|
||||
if (status === "unknown") continue; // nothing has called it yet: not up, not down
|
||||
out.add("homelab_integration_up", "gauge", "1 if the integration's latest calls are working, 0 if they are failing.", status === "ok" ? 1 : 0, { integration: i.name, type: i.type });
|
||||
}
|
||||
|
||||
// ── Maintenance ──
|
||||
const iso = now.toISOString();
|
||||
const active = await db.select({ id: maintenanceWindows.id }).from(maintenanceWindows).where(and(lte(maintenanceWindows.startedAt, iso), gt(maintenanceWindows.endsAt, iso)));
|
||||
out.add("homelab_maintenance_active_windows", "gauge", "Maintenance windows running now.", active.length);
|
||||
|
||||
// ── Things that expire ──
|
||||
for (const s of await db.select().from(secrets)) {
|
||||
const status = computeSecretStatus(s.expiryDate, s.warnDays, now);
|
||||
if (status.daysLeft !== null) out.add("homelab_secret_expiry_days", "gauge", "Days until a secret expires (negative once it has).", status.daysLeft, { name: s.name });
|
||||
}
|
||||
|
||||
// ── Inventory ──
|
||||
const power = (await db.select({ w: assets.powerWatts }).from(assets)).reduce((n, a) => n + (a.w ?? 0), 0);
|
||||
out.add("homelab_inventory_power_watts", "gauge", "Typical power draw of everything in the inventory, in watts.", power);
|
||||
|
||||
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
|
||||
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
|
||||
return out.toString();
|
||||
}
|
||||
Reference in new issue
Block a user