- Share links can open a status page: an admin picks servers, integrations and Uptime Kuma monitors and names them; the public sees only those names, Working / Not working / Being worked on / Can't tell, an optional message and the 30-day figure (share_links.kind/ title/message and status_items, migration 0033). A status address doesn't open the wall and the other way round; Uptime Kuma is asked at most every 30 seconds - Grafana dashboard built from code (download in Settings -> API and webhooks); a test checks it only uses series /metrics serves - Tests and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
280 lines
16 KiB
TypeScript
280 lines
16 KiB
TypeScript
/**
|
|
* A Grafana dashboard for the series `/metrics` serves, built here rather than kept as a file so that it can be tested against what the
|
|
* app really emits (a panel for a series that was renamed would otherwise just go blank). It is downloaded from Settings → API and
|
|
* webhooks and imported in Grafana (Dashboards → New → Import); Grafana asks which Prometheus to use, so nothing in it is tied to one
|
|
* installation.
|
|
*
|
|
* Laid out in rows — what needs looking at, servers, disks, backups, then the quieter things — each panel on a 24-column grid.
|
|
*/
|
|
|
|
const DS = { type: "prometheus", uid: "${DS_PROMETHEUS}" } as const;
|
|
|
|
type Panel = Record<string, unknown>;
|
|
|
|
interface Thresholds {
|
|
/** From the lowest: [colour, from]; the first has no lower bound. */
|
|
steps: [color: string, from: number | null][];
|
|
}
|
|
|
|
const GREEN = "green";
|
|
const YELLOW = "yellow";
|
|
const RED = "red";
|
|
const BLUE = "blue";
|
|
|
|
const thresholds = (t: Thresholds) => ({ mode: "absolute", steps: t.steps.map(([color, value]) => ({ color, value })) });
|
|
|
|
/** Red as soon as there is anything. */
|
|
const redAbove0: Thresholds = { steps: [[GREEN, null], [RED, 1]] };
|
|
const yellowAbove0: Thresholds = { steps: [[GREEN, null], [YELLOW, 1]] };
|
|
|
|
class Layout {
|
|
private y = 0;
|
|
private x = 0;
|
|
private rowHeight = 0;
|
|
private id = 1;
|
|
readonly panels: Panel[] = [];
|
|
|
|
nextId(): number {
|
|
return this.id++;
|
|
}
|
|
|
|
row(title: string): void {
|
|
this.endLine();
|
|
this.panels.push({ type: "row", id: this.nextId(), title, collapsed: false, gridPos: { h: 1, w: 24, x: 0, y: this.y }, panels: [] });
|
|
this.y += 1;
|
|
}
|
|
|
|
private endLine(): void {
|
|
if (this.x > 0) {
|
|
this.y += this.rowHeight;
|
|
this.x = 0;
|
|
this.rowHeight = 0;
|
|
}
|
|
}
|
|
|
|
add(panel: Panel, w: number, h: number): void {
|
|
if (this.x + w > 24) this.endLine();
|
|
this.panels.push({ ...panel, id: this.nextId(), gridPos: { h, w, x: this.x, y: this.y } });
|
|
this.x += w;
|
|
this.rowHeight = Math.max(this.rowHeight, h);
|
|
if (this.x >= 24) this.endLine();
|
|
}
|
|
|
|
done(): Panel[] {
|
|
this.endLine();
|
|
return this.panels;
|
|
}
|
|
}
|
|
|
|
const target = (expr: string, legend = "", extra: Record<string, unknown> = {}, refId = "A") => ({ datasource: DS, expr, legendFormat: legend, refId, ...extra });
|
|
|
|
function stat(title: string, expr: string, options: { unit?: string; thresholds?: Thresholds; legend?: string; description?: string; mappings?: unknown[]; decimals?: number; graph?: boolean } = {}): Panel {
|
|
return {
|
|
type: "stat",
|
|
title,
|
|
description: options.description ?? "",
|
|
datasource: DS,
|
|
targets: [target(expr, options.legend ?? "")],
|
|
fieldConfig: {
|
|
defaults: {
|
|
unit: options.unit ?? "none",
|
|
decimals: options.decimals,
|
|
thresholds: thresholds(options.thresholds ?? { steps: [[BLUE, null]] }),
|
|
mappings: options.mappings ?? [],
|
|
color: { mode: "thresholds" },
|
|
},
|
|
overrides: [],
|
|
},
|
|
options: { reduceOptions: { calcs: ["lastNotNull"], fields: "", values: false }, colorMode: "background", graphMode: options.graph ? "area" : "none", textMode: "auto", justifyMode: "center", orientation: "auto" },
|
|
};
|
|
}
|
|
|
|
function timeseries(title: string, queries: { expr: string; legend: string }[], options: { unit?: string; min?: number; max?: number; thresholds?: Thresholds; description?: string; stack?: boolean } = {}): Panel {
|
|
return {
|
|
type: "timeseries",
|
|
title,
|
|
description: options.description ?? "",
|
|
datasource: DS,
|
|
targets: queries.map((q, i) => target(q.expr, q.legend, {}, String.fromCharCode(65 + i))),
|
|
fieldConfig: {
|
|
defaults: {
|
|
unit: options.unit ?? "none",
|
|
min: options.min,
|
|
max: options.max,
|
|
thresholds: thresholds(options.thresholds ?? { steps: [[GREEN, null]] }),
|
|
custom: { drawStyle: "line", lineWidth: 1, fillOpacity: options.stack ? 30 : 8, showPoints: "never", spanNulls: true, stacking: { mode: options.stack ? "normal" : "none" }, thresholdsStyle: { mode: options.thresholds ? "line" : "off" } },
|
|
},
|
|
overrides: [],
|
|
},
|
|
options: { legend: { displayMode: "table", placement: "bottom", calcs: ["lastNotNull", "max"] }, tooltip: { mode: "multi", sort: "desc" } },
|
|
};
|
|
}
|
|
|
|
function bargauge(title: string, expr: string, legend: string, options: { unit?: string; min?: number; max?: number; thresholds?: Thresholds; description?: string } = {}): Panel {
|
|
return {
|
|
type: "bargauge",
|
|
title,
|
|
description: options.description ?? "",
|
|
datasource: DS,
|
|
targets: [target(expr, legend, { instant: true })],
|
|
fieldConfig: { defaults: { unit: options.unit ?? "none", min: options.min, max: options.max, thresholds: thresholds(options.thresholds ?? { steps: [[GREEN, null]] }), color: { mode: "thresholds" } }, overrides: [] },
|
|
options: { reduceOptions: { calcs: ["lastNotNull"], fields: "", values: false }, orientation: "horizontal", displayMode: "gradient", showUnfilled: true },
|
|
};
|
|
}
|
|
|
|
function table(title: string, queries: { expr: string; name: string }[], options: { unit?: string; description?: string; sortBy?: string; thresholds?: Thresholds; labels?: string[] } = {}): Panel {
|
|
// One value column per query, named, with the labels kept as columns and everything else hidden.
|
|
const rename: Record<string, string> = {};
|
|
const exclude: Record<string, boolean> = { Time: true, __name__: true };
|
|
queries.forEach((q, i) => {
|
|
rename[queries.length === 1 ? "Value" : `Value #${String.fromCharCode(65 + i)}`] = q.name;
|
|
});
|
|
return {
|
|
type: "table",
|
|
title,
|
|
description: options.description ?? "",
|
|
datasource: DS,
|
|
targets: queries.map((q, i) => target(q.expr, "", { instant: true, format: "table" }, String.fromCharCode(65 + i))),
|
|
transformations: [
|
|
{ id: "merge", options: {} },
|
|
{ id: "organize", options: { excludeByName: exclude, renameByName: { ...rename, server: "Server", mount: "Disk", device: "Device", pool: "Pool", guest: "Guest", integration: "Integration", name: "Name", status: "State" } } },
|
|
],
|
|
fieldConfig: { defaults: { unit: options.unit ?? "none", thresholds: thresholds(options.thresholds ?? { steps: [[GREEN, null]] }), custom: { align: "auto", cellOptions: { type: options.thresholds ? "color-background" : "auto" } } }, overrides: [] },
|
|
options: { showHeader: true, sortBy: options.sortBy ? [{ displayName: options.sortBy, desc: false }] : [] },
|
|
};
|
|
}
|
|
|
|
function piechart(title: string, expr: string, legend: string): Panel {
|
|
return {
|
|
type: "piechart",
|
|
title,
|
|
datasource: DS,
|
|
targets: [target(expr, legend, { instant: true })],
|
|
fieldConfig: { defaults: { unit: "none", color: { mode: "palette-classic" } }, overrides: [] },
|
|
options: { reduceOptions: { calcs: ["lastNotNull"], fields: "", values: false }, pieType: "donut", legend: { displayMode: "table", placement: "right", values: ["value"] }, displayLabels: [] },
|
|
};
|
|
}
|
|
|
|
const okFailing = [{ type: "value", options: { "0": { text: "Failing", color: RED, index: 0 }, "1": { text: "OK", color: GREEN, index: 1 } } }];
|
|
|
|
/** The dashboard, as Grafana's import takes it. */
|
|
export function buildGrafanaDashboard(): Record<string, unknown> {
|
|
const l = new Layout();
|
|
const srv = '{server=~"$server"}';
|
|
|
|
l.row("Needs attention");
|
|
l.add(stat("Critical alerts", 'sum(homelab_alerts{severity="critical"}) or vector(0)', { thresholds: redAbove0, description: "Counts every critical alert, including ones somebody has acknowledged." }), 4, 4);
|
|
l.add(stat("Warnings", 'sum(homelab_alerts{severity="warning"}) or vector(0)', { thresholds: yellowAbove0 }), 4, 4);
|
|
l.add(stat("Acknowledged", "homelab_alerts_acknowledged", { description: "Somebody has said they are on it." }), 4, 4);
|
|
l.add(stat("Open incidents", "homelab_incidents_open", { thresholds: yellowAbove0 }), 4, 4);
|
|
l.add(stat("Servers down", "count(homelab_server_up == 0) or vector(0)", { thresholds: redAbove0 }), 4, 4);
|
|
l.add(stat("Integrations failing", "count(homelab_integration_up == 0) or vector(0)", { thresholds: redAbove0 }), 4, 4);
|
|
l.add(stat("Servers up", "sum(homelab_server_up) or vector(0)", { thresholds: { steps: [[RED, null], [GREEN, 1]] } }), 4, 4);
|
|
l.add(stat("In maintenance", "homelab_maintenance_active_windows", { description: "Maintenance windows running now." }), 4, 4);
|
|
l.add(stat("Backups gone stale", 'homelab_backup_guests{status="stale"}', { thresholds: yellowAbove0, description: "Proxmox guests whose newest backup is older than its limit." }), 4, 4);
|
|
l.add(stat("Failed services", "sum(homelab_server_failed_units) or vector(0)", { thresholds: yellowAbove0 }), 4, 4);
|
|
l.add(stat("Updates waiting", "sum(homelab_server_pending_updates) or vector(0)", { thresholds: { steps: [[GREEN, null], [YELLOW, 1], [RED, 50]] } }), 4, 4);
|
|
l.add(stat("Need a restart", "sum(homelab_server_reboot_required) or vector(0)", { thresholds: yellowAbove0 }), 4, 4);
|
|
l.add(timeseries("Alerts by severity", [{ expr: "homelab_alerts", legend: "{{severity}}" }, { expr: "homelab_alerts_silenced", legend: "silenced" }, { expr: "homelab_alerts_grouped", legend: "folded under a down server" }], { description: "Raised alerts over time." }), 12, 8);
|
|
l.add(
|
|
stat("Integrations", "homelab_integration_up", { legend: "{{integration}} ({{type}})", mappings: okFailing, thresholds: { steps: [[RED, null], [GREEN, 1]] }, description: "Whether each integration's latest calls work." }),
|
|
12,
|
|
8,
|
|
);
|
|
|
|
l.row("Servers");
|
|
l.add(
|
|
table(
|
|
"Servers",
|
|
[
|
|
{ expr: `homelab_server_up${srv}`, name: "Up" },
|
|
{ expr: `homelab_server_cpu_load_percent${srv}`, name: "CPU load %" },
|
|
{ expr: `homelab_server_memory_used_percent${srv}`, name: "Memory %" },
|
|
{ expr: `homelab_server_pending_updates${srv}`, name: "Updates waiting" },
|
|
{ expr: `homelab_server_pending_security_updates${srv}`, name: "Security updates" },
|
|
{ expr: `homelab_server_failed_units${srv}`, name: "Failed services" },
|
|
{ expr: `homelab_server_agent_outdated${srv}`, name: "Agent behind" },
|
|
],
|
|
{ sortBy: "Server" },
|
|
),
|
|
24,
|
|
9,
|
|
);
|
|
l.add(timeseries("CPU load", [{ expr: `homelab_server_cpu_load_percent${srv}`, legend: "{{server}}" }], { unit: "percent", min: 0 }), 12, 8);
|
|
l.add(timeseries("Memory in use", [{ expr: `homelab_server_memory_used_percent${srv}`, legend: "{{server}}" }], { unit: "percent", min: 0, max: 100 }), 12, 8);
|
|
l.add(bargauge("Uptime, last 30 days", `homelab_server_uptime_ratio_30d${srv}`, "{{server}}", { unit: "percentunit", min: 0, max: 1, thresholds: { steps: [[RED, null], [YELLOW, 0.95], [GREEN, 0.99]] } }), 12, 8);
|
|
l.add(timeseries("Last report", [{ expr: `time() - homelab_server_last_seen_timestamp_seconds${srv}`, legend: "{{server}}" }], { unit: "s", min: 0, description: "How long since each agent last reported." }), 12, 8);
|
|
|
|
l.row("Disks");
|
|
l.add(timeseries("Disk usage", [{ expr: `homelab_server_disk_used_percent${srv}`, legend: "{{server}} {{mount}}" }], { unit: "percent", min: 0, max: 100, thresholds: { steps: [[GREEN, null], [RED, 90]] } }), 12, 9);
|
|
l.add(
|
|
table("Disks filling up", [{ expr: `sort(homelab_server_disk_days_to_full${srv})`, name: "Days to full" }], { unit: "d", thresholds: { steps: [[RED, null], [YELLOW, 7], [GREEN, 30]] }, description: "Only disks on a steady climb over the last days." }),
|
|
12,
|
|
9,
|
|
);
|
|
l.add(stat("Disk health (SMART)", `homelab_server_disk_healthy${srv}`, { legend: "{{server}} {{device}}", mappings: [{ type: "value", options: { "0": { text: "Failing", color: RED, index: 0 }, "1": { text: "Passes", color: GREEN, index: 1 } } }], thresholds: { steps: [[RED, null], [GREEN, 1]] } }), 8, 6);
|
|
l.add(bargauge("SSD life used", `homelab_server_disk_wear_percent${srv}`, "{{server}} {{device}}", { unit: "percent", min: 0, max: 100, thresholds: { steps: [[GREEN, null], [YELLOW, 80], [RED, 90]] } }), 8, 6);
|
|
l.add(timeseries("Disk temperature", [{ expr: `homelab_server_disk_temperature_celsius${srv}`, legend: "{{server}} {{device}}" }], { unit: "celsius", thresholds: { steps: [[GREEN, null], [RED, 60]] } }), 8, 6);
|
|
l.add(stat("ZFS pools", `homelab_server_zfs_pool_online${srv}`, { legend: "{{server}} / {{pool}}", mappings: [{ type: "value", options: { "0": { text: "Not online", color: RED, index: 0 }, "1": { text: "Online", color: GREEN, index: 1 } } }], thresholds: { steps: [[RED, null], [GREEN, 1]] } }), 24, 4);
|
|
|
|
l.row("Backups");
|
|
l.add(piechart("Proxmox guests by backup state", "homelab_backup_guests", "{{status}}"), 8, 8);
|
|
l.add(table("Age of each guest's newest backup", [{ expr: "sort_desc(homelab_backup_last_age_seconds)", name: "Age" }], { unit: "s", thresholds: { steps: [[GREEN, null], [YELLOW, 172800], [RED, 604800]] } }), 16, 8);
|
|
l.add(stat("Homelab Manager's own backup", "time() - homelab_selfbackup_last_success_timestamp_seconds", { unit: "s", thresholds: { steps: [[GREEN, null], [YELLOW, 129600], [RED, 172800]] }, description: "How old the newest backup of Homelab Manager's own database is." }), 6, 4);
|
|
l.add(stat("Last attempt", "homelab_selfbackup_last_attempt_ok", { mappings: okFailing, thresholds: { steps: [[RED, null], [GREEN, 1]] } }), 6, 4);
|
|
l.add(stat("Backups kept", "homelab_selfbackup_snapshots"), 6, 4);
|
|
l.add(stat("Failed backups", 'homelab_backup_guests{status="failed"}', { thresholds: redAbove0, description: "Guests whose latest backup failed, or whose newest snapshot failed verification." }), 6, 4);
|
|
|
|
l.row("Secrets, power and the app itself");
|
|
l.add(table("Secrets and certificates", [{ expr: "sort(homelab_secret_expiry_days)", name: "Days left" }], { unit: "d", thresholds: { steps: [[RED, null], [YELLOW, 14], [GREEN, 30]] }, description: "Negative once expired." }), 12, 8);
|
|
l.add(timeseries("Power draw (inventory)", [{ expr: "homelab_inventory_power_watts", legend: "typical draw" }], { unit: "watt", min: 0 }), 6, 8);
|
|
l.add(timeseries("Time to build /metrics", [{ expr: "homelab_scrape_duration_seconds", legend: "seconds" }], { unit: "s", min: 0 }), 6, 8);
|
|
|
|
return {
|
|
__inputs: [{ name: "DS_PROMETHEUS", label: "Prometheus", description: "The Prometheus that scrapes Homelab Manager", type: "datasource", pluginId: "prometheus", pluginName: "Prometheus" }],
|
|
__requires: [
|
|
{ type: "grafana", id: "grafana", name: "Grafana", version: "9.0.0" },
|
|
{ type: "datasource", id: "prometheus", name: "Prometheus", version: "1.0.0" },
|
|
{ type: "panel", id: "stat", name: "Stat", version: "" },
|
|
{ type: "panel", id: "timeseries", name: "Time series", version: "" },
|
|
{ type: "panel", id: "table", name: "Table", version: "" },
|
|
{ type: "panel", id: "bargauge", name: "Bar gauge", version: "" },
|
|
{ type: "panel", id: "piechart", name: "Pie chart", version: "" },
|
|
],
|
|
title: "Homelab Manager",
|
|
uid: "homelab-manager",
|
|
description: "Servers, disks, backups, alerts and incidents, from Homelab Manager's /metrics. Made by Homelab Manager (Settings → API and webhooks).",
|
|
tags: ["homelab", "homelab-manager"],
|
|
timezone: "browser",
|
|
editable: true,
|
|
graphTooltip: 1,
|
|
refresh: "1m",
|
|
schemaVersion: 39,
|
|
version: 1,
|
|
time: { from: "now-24h", to: "now" },
|
|
timepicker: {},
|
|
templating: {
|
|
list: [
|
|
{
|
|
name: "server",
|
|
label: "Server",
|
|
type: "query",
|
|
datasource: DS,
|
|
query: { query: "label_values(homelab_server_up, server)", refId: "server" },
|
|
definition: "label_values(homelab_server_up, server)",
|
|
includeAll: true,
|
|
allValue: ".+",
|
|
multi: true,
|
|
current: { selected: true, text: ["All"], value: ["$__all"] },
|
|
refresh: 2,
|
|
sort: 1,
|
|
hide: 0,
|
|
options: [],
|
|
},
|
|
],
|
|
},
|
|
annotations: { list: [{ builtIn: 1, datasource: { type: "grafana", uid: "-- Grafana --" }, enable: true, hide: true, iconColor: "rgba(0, 211, 255, 1)", name: "Annotations & Alerts", type: "dashboard" }] },
|
|
panels: l.done(),
|
|
};
|
|
}
|