/** * A Grafana dashboard for the series `/metrics` serves, built here rather than kept as a file so that it can be tested against what the * app really emits (a panel for a series that was renamed would otherwise just go blank). It is downloaded from Settings → API and * webhooks and imported in Grafana (Dashboards → New → Import); Grafana asks which Prometheus to use, so nothing in it is tied to one * installation. * * Laid out in rows — what needs looking at, servers, disks, backups, then the quieter things — each panel on a 24-column grid. */ const DS = { type: "prometheus", uid: "${DS_PROMETHEUS}" } as const; type Panel = Record; interface Thresholds { /** From the lowest: [colour, from]; the first has no lower bound. */ steps: [color: string, from: number | null][]; } const GREEN = "green"; const YELLOW = "yellow"; const RED = "red"; const BLUE = "blue"; const thresholds = (t: Thresholds) => ({ mode: "absolute", steps: t.steps.map(([color, value]) => ({ color, value })) }); /** Red as soon as there is anything. */ const redAbove0: Thresholds = { steps: [[GREEN, null], [RED, 1]] }; const yellowAbove0: Thresholds = { steps: [[GREEN, null], [YELLOW, 1]] }; class Layout { private y = 0; private x = 0; private rowHeight = 0; private id = 1; readonly panels: Panel[] = []; nextId(): number { return this.id++; } row(title: string): void { this.endLine(); this.panels.push({ type: "row", id: this.nextId(), title, collapsed: false, gridPos: { h: 1, w: 24, x: 0, y: this.y }, panels: [] }); this.y += 1; } private endLine(): void { if (this.x > 0) { this.y += this.rowHeight; this.x = 0; this.rowHeight = 0; } } add(panel: Panel, w: number, h: number): void { if (this.x + w > 24) this.endLine(); this.panels.push({ ...panel, id: this.nextId(), gridPos: { h, w, x: this.x, y: this.y } }); this.x += w; this.rowHeight = Math.max(this.rowHeight, h); if (this.x >= 24) this.endLine(); } done(): Panel[] { this.endLine(); return this.panels; } } const target = (expr: string, legend = "", extra: Record = {}, refId = "A") => ({ datasource: DS, expr, legendFormat: legend, refId, ...extra }); function stat(title: string, expr: string, options: { unit?: string; thresholds?: Thresholds; legend?: string; description?: string; mappings?: unknown[]; decimals?: number; graph?: boolean } = {}): Panel { return { type: "stat", title, description: options.description ?? "", datasource: DS, targets: [target(expr, options.legend ?? "")], fieldConfig: { defaults: { unit: options.unit ?? "none", decimals: options.decimals, thresholds: thresholds(options.thresholds ?? { steps: [[BLUE, null]] }), mappings: options.mappings ?? [], color: { mode: "thresholds" }, }, overrides: [], }, options: { reduceOptions: { calcs: ["lastNotNull"], fields: "", values: false }, colorMode: "background", graphMode: options.graph ? "area" : "none", textMode: "auto", justifyMode: "center", orientation: "auto" }, }; } function timeseries(title: string, queries: { expr: string; legend: string }[], options: { unit?: string; min?: number; max?: number; thresholds?: Thresholds; description?: string; stack?: boolean } = {}): Panel { return { type: "timeseries", title, description: options.description ?? "", datasource: DS, targets: queries.map((q, i) => target(q.expr, q.legend, {}, String.fromCharCode(65 + i))), fieldConfig: { defaults: { unit: options.unit ?? "none", min: options.min, max: options.max, thresholds: thresholds(options.thresholds ?? { steps: [[GREEN, null]] }), custom: { drawStyle: "line", lineWidth: 1, fillOpacity: options.stack ? 30 : 8, showPoints: "never", spanNulls: true, stacking: { mode: options.stack ? "normal" : "none" }, thresholdsStyle: { mode: options.thresholds ? "line" : "off" } }, }, overrides: [], }, options: { legend: { displayMode: "table", placement: "bottom", calcs: ["lastNotNull", "max"] }, tooltip: { mode: "multi", sort: "desc" } }, }; } function bargauge(title: string, expr: string, legend: string, options: { unit?: string; min?: number; max?: number; thresholds?: Thresholds; description?: string } = {}): Panel { return { type: "bargauge", title, description: options.description ?? "", datasource: DS, targets: [target(expr, legend, { instant: true })], fieldConfig: { defaults: { unit: options.unit ?? "none", min: options.min, max: options.max, thresholds: thresholds(options.thresholds ?? { steps: [[GREEN, null]] }), color: { mode: "thresholds" } }, overrides: [] }, options: { reduceOptions: { calcs: ["lastNotNull"], fields: "", values: false }, orientation: "horizontal", displayMode: "gradient", showUnfilled: true }, }; } function table(title: string, queries: { expr: string; name: string }[], options: { unit?: string; description?: string; sortBy?: string; thresholds?: Thresholds; labels?: string[] } = {}): Panel { // One value column per query, named, with the labels kept as columns and everything else hidden. const rename: Record = {}; const exclude: Record = { Time: true, __name__: true }; queries.forEach((q, i) => { rename[queries.length === 1 ? "Value" : `Value #${String.fromCharCode(65 + i)}`] = q.name; }); return { type: "table", title, description: options.description ?? "", datasource: DS, targets: queries.map((q, i) => target(q.expr, "", { instant: true, format: "table" }, String.fromCharCode(65 + i))), transformations: [ { id: "merge", options: {} }, { id: "organize", options: { excludeByName: exclude, renameByName: { ...rename, server: "Server", mount: "Disk", device: "Device", pool: "Pool", guest: "Guest", integration: "Integration", name: "Name", status: "State" } } }, ], fieldConfig: { defaults: { unit: options.unit ?? "none", thresholds: thresholds(options.thresholds ?? { steps: [[GREEN, null]] }), custom: { align: "auto", cellOptions: { type: options.thresholds ? "color-background" : "auto" } } }, overrides: [] }, options: { showHeader: true, sortBy: options.sortBy ? [{ displayName: options.sortBy, desc: false }] : [] }, }; } function piechart(title: string, expr: string, legend: string): Panel { return { type: "piechart", title, datasource: DS, targets: [target(expr, legend, { instant: true })], fieldConfig: { defaults: { unit: "none", color: { mode: "palette-classic" } }, overrides: [] }, options: { reduceOptions: { calcs: ["lastNotNull"], fields: "", values: false }, pieType: "donut", legend: { displayMode: "table", placement: "right", values: ["value"] }, displayLabels: [] }, }; } const okFailing = [{ type: "value", options: { "0": { text: "Failing", color: RED, index: 0 }, "1": { text: "OK", color: GREEN, index: 1 } } }]; /** The dashboard, as Grafana's import takes it. */ export function buildGrafanaDashboard(): Record { const l = new Layout(); const srv = '{server=~"$server"}'; l.row("Needs attention"); l.add(stat("Critical alerts", 'sum(homelab_alerts{severity="critical"}) or vector(0)', { thresholds: redAbove0, description: "Counts every critical alert, including ones somebody has acknowledged." }), 4, 4); l.add(stat("Warnings", 'sum(homelab_alerts{severity="warning"}) or vector(0)', { thresholds: yellowAbove0 }), 4, 4); l.add(stat("Acknowledged", "homelab_alerts_acknowledged", { description: "Somebody has said they are on it." }), 4, 4); l.add(stat("Open incidents", "homelab_incidents_open", { thresholds: yellowAbove0 }), 4, 4); l.add(stat("Servers down", "count(homelab_server_up == 0) or vector(0)", { thresholds: redAbove0 }), 4, 4); l.add(stat("Integrations failing", "count(homelab_integration_up == 0) or vector(0)", { thresholds: redAbove0 }), 4, 4); l.add(stat("Servers up", "sum(homelab_server_up) or vector(0)", { thresholds: { steps: [[RED, null], [GREEN, 1]] } }), 4, 4); l.add(stat("In maintenance", "homelab_maintenance_active_windows", { description: "Maintenance windows running now." }), 4, 4); l.add(stat("Backups gone stale", 'homelab_backup_guests{status="stale"}', { thresholds: yellowAbove0, description: "Proxmox guests whose newest backup is older than its limit." }), 4, 4); l.add(stat("Failed services", "sum(homelab_server_failed_units) or vector(0)", { thresholds: yellowAbove0 }), 4, 4); l.add(stat("Updates waiting", "sum(homelab_server_pending_updates) or vector(0)", { thresholds: { steps: [[GREEN, null], [YELLOW, 1], [RED, 50]] } }), 4, 4); l.add(stat("Need a restart", "sum(homelab_server_reboot_required) or vector(0)", { thresholds: yellowAbove0 }), 4, 4); l.add(timeseries("Alerts by severity", [{ expr: "homelab_alerts", legend: "{{severity}}" }, { expr: "homelab_alerts_silenced", legend: "silenced" }, { expr: "homelab_alerts_grouped", legend: "folded under a down server" }], { description: "Raised alerts over time." }), 12, 8); l.add( stat("Integrations", "homelab_integration_up", { legend: "{{integration}} ({{type}})", mappings: okFailing, thresholds: { steps: [[RED, null], [GREEN, 1]] }, description: "Whether each integration's latest calls work." }), 12, 8, ); l.row("Servers"); l.add( table( "Servers", [ { expr: `homelab_server_up${srv}`, name: "Up" }, { expr: `homelab_server_cpu_load_percent${srv}`, name: "CPU load %" }, { expr: `homelab_server_memory_used_percent${srv}`, name: "Memory %" }, { expr: `homelab_server_pending_updates${srv}`, name: "Updates waiting" }, { expr: `homelab_server_pending_security_updates${srv}`, name: "Security updates" }, { expr: `homelab_server_failed_units${srv}`, name: "Failed services" }, { expr: `homelab_server_agent_outdated${srv}`, name: "Agent behind" }, ], { sortBy: "Server" }, ), 24, 9, ); l.add(timeseries("CPU load", [{ expr: `homelab_server_cpu_load_percent${srv}`, legend: "{{server}}" }], { unit: "percent", min: 0 }), 12, 8); l.add(timeseries("Memory in use", [{ expr: `homelab_server_memory_used_percent${srv}`, legend: "{{server}}" }], { unit: "percent", min: 0, max: 100 }), 12, 8); l.add(bargauge("Uptime, last 30 days", `homelab_server_uptime_ratio_30d${srv}`, "{{server}}", { unit: "percentunit", min: 0, max: 1, thresholds: { steps: [[RED, null], [YELLOW, 0.95], [GREEN, 0.99]] } }), 12, 8); l.add(timeseries("Last report", [{ expr: `time() - homelab_server_last_seen_timestamp_seconds${srv}`, legend: "{{server}}" }], { unit: "s", min: 0, description: "How long since each agent last reported." }), 12, 8); l.row("Disks"); l.add(timeseries("Disk usage", [{ expr: `homelab_server_disk_used_percent${srv}`, legend: "{{server}} {{mount}}" }], { unit: "percent", min: 0, max: 100, thresholds: { steps: [[GREEN, null], [RED, 90]] } }), 12, 9); l.add( table("Disks filling up", [{ expr: `sort(homelab_server_disk_days_to_full${srv})`, name: "Days to full" }], { unit: "d", thresholds: { steps: [[RED, null], [YELLOW, 7], [GREEN, 30]] }, description: "Only disks on a steady climb over the last days." }), 12, 9, ); l.add(stat("Disk health (SMART)", `homelab_server_disk_healthy${srv}`, { legend: "{{server}} {{device}}", mappings: [{ type: "value", options: { "0": { text: "Failing", color: RED, index: 0 }, "1": { text: "Passes", color: GREEN, index: 1 } } }], thresholds: { steps: [[RED, null], [GREEN, 1]] } }), 8, 6); l.add(bargauge("SSD life used", `homelab_server_disk_wear_percent${srv}`, "{{server}} {{device}}", { unit: "percent", min: 0, max: 100, thresholds: { steps: [[GREEN, null], [YELLOW, 80], [RED, 90]] } }), 8, 6); l.add(timeseries("Disk temperature", [{ expr: `homelab_server_disk_temperature_celsius${srv}`, legend: "{{server}} {{device}}" }], { unit: "celsius", thresholds: { steps: [[GREEN, null], [RED, 60]] } }), 8, 6); l.add(stat("ZFS pools", `homelab_server_zfs_pool_online${srv}`, { legend: "{{server}} / {{pool}}", mappings: [{ type: "value", options: { "0": { text: "Not online", color: RED, index: 0 }, "1": { text: "Online", color: GREEN, index: 1 } } }], thresholds: { steps: [[RED, null], [GREEN, 1]] } }), 24, 4); l.row("Backups"); l.add(piechart("Proxmox guests by backup state", "homelab_backup_guests", "{{status}}"), 8, 8); l.add(table("Age of each guest's newest backup", [{ expr: "sort_desc(homelab_backup_last_age_seconds)", name: "Age" }], { unit: "s", thresholds: { steps: [[GREEN, null], [YELLOW, 172800], [RED, 604800]] } }), 16, 8); l.add(stat("Homelab Manager's own backup", "time() - homelab_selfbackup_last_success_timestamp_seconds", { unit: "s", thresholds: { steps: [[GREEN, null], [YELLOW, 129600], [RED, 172800]] }, description: "How old the newest backup of Homelab Manager's own database is." }), 6, 4); l.add(stat("Last attempt", "homelab_selfbackup_last_attempt_ok", { mappings: okFailing, thresholds: { steps: [[RED, null], [GREEN, 1]] } }), 6, 4); l.add(stat("Backups kept", "homelab_selfbackup_snapshots"), 6, 4); l.add(stat("Failed backups", 'homelab_backup_guests{status="failed"}', { thresholds: redAbove0, description: "Guests whose latest backup failed, or whose newest snapshot failed verification." }), 6, 4); l.row("Secrets, power and the app itself"); l.add(table("Secrets and certificates", [{ expr: "sort(homelab_secret_expiry_days)", name: "Days left" }], { unit: "d", thresholds: { steps: [[RED, null], [YELLOW, 14], [GREEN, 30]] }, description: "Negative once expired." }), 12, 8); l.add(timeseries("Power draw (inventory)", [{ expr: "homelab_inventory_power_watts", legend: "typical draw" }], { unit: "watt", min: 0 }), 6, 8); l.add(timeseries("Time to build /metrics", [{ expr: "homelab_scrape_duration_seconds", legend: "seconds" }], { unit: "s", min: 0 }), 6, 8); return { __inputs: [{ name: "DS_PROMETHEUS", label: "Prometheus", description: "The Prometheus that scrapes Homelab Manager", type: "datasource", pluginId: "prometheus", pluginName: "Prometheus" }], __requires: [ { type: "grafana", id: "grafana", name: "Grafana", version: "9.0.0" }, { type: "datasource", id: "prometheus", name: "Prometheus", version: "1.0.0" }, { type: "panel", id: "stat", name: "Stat", version: "" }, { type: "panel", id: "timeseries", name: "Time series", version: "" }, { type: "panel", id: "table", name: "Table", version: "" }, { type: "panel", id: "bargauge", name: "Bar gauge", version: "" }, { type: "panel", id: "piechart", name: "Pie chart", version: "" }, ], title: "Homelab Manager", uid: "homelab-manager", description: "Servers, disks, backups, alerts and incidents, from Homelab Manager's /metrics. Made by Homelab Manager (Settings → API and webhooks).", tags: ["homelab", "homelab-manager"], timezone: "browser", editable: true, graphTooltip: 1, refresh: "1m", schemaVersion: 39, version: 1, time: { from: "now-24h", to: "now" }, timepicker: {}, templating: { list: [ { name: "server", label: "Server", type: "query", datasource: DS, query: { query: "label_values(homelab_server_up, server)", refId: "server" }, definition: "label_values(homelab_server_up, server)", includeAll: true, allValue: ".+", multi: true, current: { selected: true, text: ["All"], value: ["$__all"] }, refresh: 2, sort: 1, hide: 0, options: [], }, ], }, annotations: { list: [{ builtIn: 1, datasource: { type: "grafana", uid: "-- Grafana --" }, enable: true, hide: true, iconColor: "rgba(0, 211, 255, 1)", name: "Annotations & Alerts", type: "dashboard" }] }, panels: l.done(), }; }