Add runbooks, backup coverage, inventory, roles from groups, Prometheus metrics and event webhooks

- Runbooks: notes per server/integration, shown next to matching alerts
- Backup coverage page across Proxmox and PBS, with a digest section
- Hardware inventory with warranty, power and running cost
- Authentik group to role mapping with preview and safeguards
- /metrics (bearer token) and signed outgoing event webhooks
- Docs: README, DATABASE, NOTIFICATIONS, ROLES and new API.md

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-17 20:41:00 +02:00
1 parent c36e2092c5
commit c17e9aa140
55 files changed
+7177 -27

No files matched your search

+219
View File
@@ -0,0 +1,219 @@
/**
* Backup coverage: for every Proxmox guest, is it in a backup job, when was it last backed up, and did that go well — across Proxmox's
* own backup tasks and Proxmox Backup Server. It answers "what isn't backed up?" in one place, instead of an alert per gap.
*
* Where the answer comes from, and what it can't say:
* - "In a job" is worked out from the backup jobs (the same rule the backup alerts use).
* - "Last backup" is the newest snapshot of that guest on a Backup Server, or the last successful Proxmox backup task that names that
* guest. A task from an "all guests" job doesn't name its guests (Proxmox keeps those in the task's log), so for guests backed up to
* something other than a Backup Server, a job that covers them with no way of seeing when it last ran reads as "unknown" — said so,
* never guessed at.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, servers } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createPbsAdapter, type PbsBackupGroup } from "../integrations/pbs/adapter.js";
import { createProxmoxAdapter, guestsWithoutBackupCoverage, type ProxmoxBackupJob, type ProxmoxBackupTask, type ProxmoxGuest } from "../integrations/proxmox/adapter.js";
export type CoverageStatus = "ok" | "stale" | "failed" | "none" | "unknown";
export const DEFAULT_STALE_HOURS = 48;
export interface CoverageGuest {
integrationId: number;
integrationName: string;
node: string;
vmid: number;
name: string;
type: "qemu" | "lxc";
/** The guest's power state, as Proxmox says. */
power: string;
inJob: boolean;
/** ISO — the newest backup found, from either source. */
lastBackupAt: string | null;
lastSource: "pbs" | "task" | null;
status: CoverageStatus;
/** Why it is "failed", when it is. */
failure: string | null;
/** The server in this app that is this guest, if one is linked to it. */
server: { id: number; name: string } | null;
}
export interface ProxmoxSource {
integrationId: number;
integrationName: string;
guests: ProxmoxGuest[];
jobs: ProxmoxBackupJob[];
tasks: ProxmoxBackupTask[];
}
export interface PbsSource {
integrationName: string;
groups: PbsBackupGroup[];
}
export interface SourceStatus {
kind: "proxmox" | "pbs";
name: string;
ok: boolean;
error?: string;
}
export interface LinkedServer {
id: number;
name: string;
proxmoxIntegrationId: number | null;
proxmoxVmid: number | null;
}
export interface CoverageReport {
guests: CoverageGuest[];
counts: Record<CoverageStatus, number> & { total: number };
sources: SourceStatus[];
staleHours: number;
generatedAt: string;
}
/**
* Pure: the rows, from what the sources said. A guest that is in no job and has no backup at all is "none"; one with a failed latest
* run or a snapshot that failed verification is "failed"; otherwise it is "ok" or "stale" by the age of its newest backup; and one in a job
* with nothing to say when it last ran is "unknown".
*/
export function buildCoverage(input: { proxmox: ProxmoxSource[]; pbs: PbsSource[]; servers: LinkedServer[]; now: Date; staleHours?: number }): CoverageGuest[] {
const staleMs = (input.staleHours ?? DEFAULT_STALE_HOURS) * 3600_000;
const nowMs = input.now.getTime();
// The newest snapshot of each guest, across every Backup Server (guests are told apart there by "vm/100" or "ct/101").
const snapshots = new Map<string, PbsBackupGroup>();
for (const src of input.pbs) {
for (const g of src.groups) {
const key = `${g.backupType}/${g.backupId}`;
const seen = snapshots.get(key);
if (!seen || g.backupTime > seen.backupTime) snapshots.set(key, g);
}
}
const out: CoverageGuest[] = [];
for (const src of input.proxmox) {
const uncovered = new Set(guestsWithoutBackupCoverage(src.guests, src.jobs).map((g) => g.vmid));
for (const g of src.guests) {
const forGuest = src.tasks.filter((t) => t.guestId === String(g.vmid)).sort((a, b) => b.startTime.localeCompare(a.startTime));
const lastOkTask = forGuest.find((t) => t.ok);
const latestTask = forGuest[0];
const snap = snapshots.get(`${g.type === "qemu" ? "vm" : "ct"}/${g.vmid}`);
const candidates: { at: number; source: "pbs" | "task" }[] = [];
if (snap) candidates.push({ at: snap.backupTime * 1000, source: "pbs" });
if (lastOkTask) candidates.push({ at: Date.parse(lastOkTask.endTime ?? lastOkTask.startTime), source: "task" });
const newest = candidates.sort((a, b) => b.at - a.at)[0];
let failure: string | null = null;
if (snap?.verification === "failed" && newest?.source === "pbs") failure = "the newest snapshot failed verification";
else if (latestTask && !latestTask.ok && latestTask.status !== "running" && (!newest || Date.parse(latestTask.startTime) > newest.at)) failure = `the latest backup didn't succeed (${latestTask.status})`;
const inJob = !uncovered.has(g.vmid);
let status: CoverageStatus;
if (failure) status = "failed";
else if (newest) status = nowMs - newest.at > staleMs ? "stale" : "ok";
else status = inJob ? "unknown" : "none";
const linked = input.servers.find((s) => s.proxmoxIntegrationId === src.integrationId && s.proxmoxVmid === g.vmid);
out.push({
integrationId: src.integrationId,
integrationName: src.integrationName,
node: g.node,
vmid: g.vmid,
name: g.name,
type: g.type,
power: g.status,
inJob,
lastBackupAt: newest ? new Date(newest.at).toISOString() : null,
lastSource: newest ? newest.source : null,
status,
failure,
server: linked ? { id: linked.id, name: linked.name } : null,
});
}
}
return out;
}
export function summarize(guests: CoverageGuest[]): CoverageReport["counts"] {
const counts = { ok: 0, stale: 0, failed: 0, none: 0, unknown: 0, total: guests.length };
for (const g of guests) counts[g.status]++;
return counts;
}
// ─── Reading the sources ────────────────────────────────────────────────────
interface Loaded {
proxmox: ProxmoxSource[];
pbs: PbsSource[];
sources: SourceStatus[];
}
/** How long what was read from Proxmox and Backup Server is reused: a page refreshed a few times asks them once. */
const CACHE_MS = 60_000;
let cache: { at: number; loaded: Loaded } | null = null;
let loaderOverride: (() => Promise<Loaded>) | null = null;
/** For tests: read from this instead of the real integrations. */
export function setCoverageLoaderForTests(loader: (() => Promise<Loaded>) | null): void {
loaderOverride = loader;
cache = null;
}
async function loadSources(): Promise<Loaded> {
if (loaderOverride) return loaderOverride();
const loaded: Loaded = { proxmox: [], pbs: [], sources: [] };
const px = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "proxmox"), eq(integrations.enabled, true)));
await Promise.all(
px.map(async (row) => {
try {
const cfg = await loadIntegrationConfig(row.id);
if (!cfg) return;
const adapter = createProxmoxAdapter(cfg.config as any);
const [guests, jobs, tasks] = await Promise.all([adapter.listGuests(), adapter.listBackupJobs(), adapter.listRecentBackupTasks(50)]);
loaded.proxmox.push({ integrationId: row.id, integrationName: row.name, guests, jobs, tasks });
loaded.sources.push({ kind: "proxmox", name: row.name, ok: true });
} catch (err) {
loaded.sources.push({ kind: "proxmox", name: row.name, ok: false, error: err instanceof Error ? err.message : String(err) });
}
}),
);
const pbs = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "pbs"), eq(integrations.enabled, true)));
await Promise.all(
pbs.map(async (row) => {
try {
const cfg = await loadIntegrationConfig(row.id);
if (!cfg) return;
const datastores = await createPbsAdapter(cfg.config as any).listDatastores();
const readable = datastores.filter((d) => d.error === null);
loaded.pbs.push({ integrationName: row.name, groups: readable.flatMap((d) => d.groups) });
const unreadable = datastores.filter((d) => d.error !== null);
loaded.sources.push(unreadable.length > 0 ? { kind: "pbs", name: row.name, ok: false, error: `couldn't read ${unreadable.map((d) => d.name).join(", ")}` } : { kind: "pbs", name: row.name, ok: true });
} catch (err) {
loaded.sources.push({ kind: "pbs", name: row.name, ok: false, error: err instanceof Error ? err.message : String(err) });
}
}),
);
loaded.sources.sort((a, b) => a.name.localeCompare(b.name));
return loaded;
}
export async function getBackupCoverage(options: { staleHours?: number; refresh?: boolean; now?: Date } = {}): Promise<CoverageReport> {
const now = options.now ?? new Date();
const staleHours = options.staleHours ?? DEFAULT_STALE_HOURS;
if (!cache || options.refresh || now.getTime() - cache.at > CACHE_MS || loaderOverride) {
cache = { at: now.getTime(), loaded: await loadSources() };
}
const serverRows = await db.select({ id: servers.id, name: servers.name, proxmoxIntegrationId: servers.proxmoxIntegrationId, proxmoxVmid: servers.proxmoxVmid }).from(servers);
const guests = buildCoverage({ proxmox: cache.loaded.proxmox, pbs: cache.loaded.pbs, servers: serverRows, now, staleHours }).sort(
(a, b) => a.name.localeCompare(b.name) || a.vmid - b.vmid,
);
return { guests, counts: summarize(guests), sources: cache.loaded.sources, staleHours, generatedAt: now.toISOString() };
}
+105
View File
@@ -0,0 +1,105 @@
/**
* Event webhooks: tell another system when something happens. Each webhook is a URL with the events it wants; when one happens it is
* sent a JSON message, signed with the webhook's secret so the receiver can tell it came from here. This is separate from the chat-style
* webhook under Settings → Notifications, which carries the same sentence a person would read.
*
* alert.opened / alert.resolved a problem began or ended — on the same terms as a notification (a problem under maintenance,
* or folded under another, isn't announced until it stands alone)
* maintenance.started / .ended a maintenance window began or finished, however it was made
*
* Delivery is best-effort and never holds anything up: one try, one more if that fails, then the result is noted on the webhook (shown in
* Settings) and forgotten.
*/
import { createHmac, randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { eventWebhooks, webhookEvents, type WebhookEvent } from "../db/schema.js";
import { decryptSecret, encryptSecret } from "../crypto.js";
import { env } from "../env.js";
export type EventWebhookRow = typeof eventWebhooks.$inferSelect;
const ENCRYPTED_PREFIX = "enc:v1:";
const TIMEOUT_MS = 5000;
const RETRY_AFTER_MS = 2000;
/** The stored form of a secret: encrypted when an encryption key is set, as the notification channels' secrets are. */
export function sealSecret(plain: string): string {
return env.credentialsEncryptionEnabled ? ENCRYPTED_PREFIX + encryptSecret(plain) : plain;
}
function openSecret(stored: string | null): string | null {
if (!stored) return null;
if (!stored.startsWith(ENCRYPTED_PREFIX)) return stored;
try {
return decryptSecret(stored.slice(ENCRYPTED_PREFIX.length));
} catch {
return null;
}
}
export function parseEvents(stored: string): WebhookEvent[] {
try {
const v = JSON.parse(stored);
return Array.isArray(v) ? v.filter((e): e is WebhookEvent => (webhookEvents as readonly string[]).includes(e)) : [];
} catch {
return [];
}
}
/** The signature a receiver should compute over the exact body it got: HMAC-SHA256 with the secret, as hex, prefixed "sha256=". */
export function signBody(secret: string, body: string): string {
return `sha256=${createHmac("sha256", secret).update(body).digest("hex")}`;
}
export interface Delivery {
ok: boolean;
/** "200", or what went wrong. */
result: string;
}
async function post(row: EventWebhookRow, event: string, body: string, deliveryId: string): Promise<Delivery> {
const secret = openSecret(row.secret);
const headers: Record<string, string> = {
"Content-Type": "application/json",
"User-Agent": "HomelabManager-Webhook",
"X-HLM-Event": event,
"X-HLM-Delivery": deliveryId,
};
if (secret) headers["X-HLM-Signature"] = signBody(secret, body);
try {
const res = await fetch(row.url, { method: "POST", headers, body, signal: AbortSignal.timeout(TIMEOUT_MS), redirect: "manual" });
return { ok: res.status >= 200 && res.status < 300, result: String(res.status) };
} catch (err) {
const name = err instanceof Error ? err.name : "";
return { ok: false, result: name === "TimeoutError" ? "timed out" : err instanceof Error ? err.message.slice(0, 120) : "failed" };
}
}
/** One message to one webhook, retried once if it didn't go through. The outcome is written on the webhook. */
export async function deliver(row: EventWebhookRow, event: string, data: unknown, options: { retryAfterMs?: number } = {}): Promise<Delivery> {
const deliveryId = randomUUID();
const body = JSON.stringify({ event, id: deliveryId, createdAt: new Date().toISOString(), source: "homelab-manager", data });
let outcome = await post(row, event, body, deliveryId);
if (!outcome.ok) {
await new Promise((r) => setTimeout(r, options.retryAfterMs ?? RETRY_AFTER_MS));
outcome = await post(row, event, body, deliveryId);
}
try {
await db.update(eventWebhooks).set({ lastAttemptAt: new Date().toISOString(), lastResult: outcome.result }).where(eq(eventWebhooks.id, row.id));
} catch (err) {
console.error("[webhooks] couldn't note the result:", err);
}
return outcome;
}
/**
* Sends an event to every enabled webhook that wants it. Returns at once — delivery carries on in the background and can never delay or
* fail what caused the event.
*/
export function emitEvent(event: WebhookEvent, data: unknown): void {
void (async () => {
const rows = await db.select().from(eventWebhooks).where(eq(eventWebhooks.enabled, true));
await Promise.all(rows.filter((r) => parseEvents(r.events).includes(event)).map((r) => deliver(r, event, data)));
})().catch((err) => console.error("[webhooks] couldn't send an event:", err));
}
+42
View File
@@ -0,0 +1,42 @@
/**
* The small calculations around the hardware inventory: how long a warranty has left, and what a machine's power draw costs to run.
* Kept apart from the routes so the rules can be tested exactly.
*/
export type WarrantyStatus = "none" | "ok" | "expiring" | "expired";
/** A warranty that ends within this many days is "expiring". */
export const WARRANTY_WARN_DAYS = 90;
/** Whether a `YYYY-MM-DD` is a real calendar date. */
export function isRealDate(value: string): boolean {
if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false;
const t = Date.parse(`${value}T00:00:00Z`);
return Number.isFinite(t) && new Date(t).toISOString().slice(0, 10) === value;
}
/** Days from today (UTC) to the end of a warranty; negative once it has ended. Null with no warranty date. */
export function warrantyDaysLeft(warrantyUntil: string | null, now: Date = new Date()): number | null {
if (!warrantyUntil || !isRealDate(warrantyUntil)) return null;
const end = Date.parse(`${warrantyUntil}T00:00:00Z`);
const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate());
return Math.round((end - today) / 86_400_000);
}
export function warrantyStatus(warrantyUntil: string | null, now: Date = new Date(), warnDays: number = WARRANTY_WARN_DAYS): WarrantyStatus {
const days = warrantyDaysLeft(warrantyUntil, now);
if (days === null) return "none";
if (days < 0) return "expired";
return days <= warnDays ? "expiring" : "ok";
}
/** Kilowatt-hours a steady draw uses in a year. */
export function yearlyKwh(watts: number): number {
return Math.round((watts / 1000) * 24 * 365 * 100) / 100;
}
/** What a year of a steady draw costs at a price per kWh, rounded to whole units of currency. Null when there is no draw or no price. */
export function yearlyCost(watts: number | null, pricePerKwh: number | null): number | null {
if (watts === null || watts <= 0 || pricePerKwh === null || !(pricePerKwh > 0)) return null;
return Math.round(yearlyKwh(watts) * pricePerKwh);
}
+53
View File
@@ -0,0 +1,53 @@
/**
* Turns maintenance windows beginning and ending into events, whatever made the window — by hand, in bulk, from a recurring schedule or
* mirrored from Uptime Kuma, and whether it ended on its own or was ended early. It simply watches which windows are running: one that
* wasn't a moment ago is "started", one that was and isn't now is "ended". After a restart it starts from what is running then, so nothing
* is announced twice or announced for a window that began before.
*/
import { listActiveWindows, describeTarget, type ActiveWindow } from "./maintenance.js";
import { emitEvent } from "./eventWebhooks.js";
const INTERVAL_MS = 30_000;
let known: Map<number, ActiveWindow> | null = null;
let timer: ReturnType<typeof setInterval> | null = null;
async function describe(w: ActiveWindow) {
const target = await describeTarget(w.targetType, w.targetId);
return { id: w.id, targetType: w.targetType, targetId: w.targetId, targetName: target?.name ?? null, reason: w.reason, startedAt: w.startedAt, endsAt: w.endsAt, source: w.source };
}
/** One look. Returns what it announced, for the tests. */
export async function checkMaintenanceEvents(now: Date = new Date()): Promise<{ started: number[]; ended: number[] }> {
const active = new Map((await listActiveWindows(now)).map((w) => [w.id, w]));
const started: number[] = [];
const ended: number[] = [];
if (known !== null) {
for (const [id, w] of active) {
if (!known.has(id)) {
started.push(id);
emitEvent("maintenance.started", { window: await describe(w) });
}
}
for (const [id, w] of known) {
if (!active.has(id)) {
ended.push(id);
// The window as it was last seen (one ended early has had its end brought forward since), and when it was noticed to be over.
emitEvent("maintenance.ended", { window: { ...(await describe(w)), endedAt: now.toISOString() } });
}
}
}
known = active;
return { started, ended };
}
/** For tests: forget what was running, so the next look is a first look. */
export function resetMaintenanceEventsForTests(): void {
known = null;
}
export function initMaintenanceEvents(): void {
if (timer) return;
void checkMaintenanceEvents().catch((err) => console.error("[maintenanceEvents]", err));
timer = setInterval(() => void checkMaintenanceEvents().catch((err) => console.error("[maintenanceEvents]", err)), INTERVAL_MS);
}
+131
View File
@@ -0,0 +1,131 @@
/**
* What /metrics says, in Prometheus's text format: servers (reporting or not, load, memory, disks, updates, agent version), alerts by
* severity, integrations, maintenance, things that expire. It reads what the app already knows — nothing here calls out to a server or an
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
*/
import { db } from "../db/client.js";
import { assets, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
import { env } from "../env.js";
import { getAlerts } from "./alerts.js";
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
import { getSourceStatus } from "./integrationHealthMonitor.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { computeSecretStatus } from "./secretStatus.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
import { and, gt, lte } from "drizzle-orm";
/** A label value as Prometheus wants it: backslash, double quote and newline escaped. */
export function escapeLabel(value: string): string {
return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
}
/** A metric name's HELP text may not contain a raw newline either. */
const help = (text: string) => text.replace(/\\/g, "\\\\").replace(/\n/g, "\\n");
type Labels = Record<string, string>;
class Exposition {
private lines: string[] = [];
private declared = new Set<string>();
/** One sample. The HELP and TYPE lines for a metric are written the first time it appears. */
add(name: string, type: "gauge" | "counter", description: string, value: number, labels: Labels = {}): void {
if (!Number.isFinite(value)) return;
if (!this.declared.has(name)) {
this.declared.add(name);
this.lines.push(`# HELP ${name} ${help(description)}`, `# TYPE ${name} ${type}`);
}
const parts = Object.entries(labels).map(([k, v]) => `${k}="${escapeLabel(v)}"`);
this.lines.push(`${name}${parts.length ? `{${parts.join(",")}}` : ""} ${Number.isInteger(value) ? value : Number(value.toFixed(4))}`);
}
toString(): string {
return `${this.lines.join("\n")}\n`;
}
}
export async function buildMetrics(now: Date = new Date()): Promise<string> {
const started = Date.now();
const out = new Exposition();
const settings = await getSettings();
const nowSec = Math.floor(now.getTime() / 1000);
out.add("homelab_info", "gauge", "Always 1; the labels say which installation this is.", 1, { environment: env.environment?.name || "" });
// ── Servers ──
const latest = latestAgentVersions();
const serverRows = await db.select().from(servers);
const offlineAfterMs = settings.healthChecks.serverOfflineMinutes * 60_000;
for (const s of serverRows) {
if (!s.lastSeenAt) continue; // never reported: nothing to say about it
const server = s.name;
const seen = Date.parse(s.lastSeenAt);
out.add("homelab_server_up", "gauge", "1 if the server's agent has reported within the offline threshold, else 0.", now.getTime() - seen <= offlineAfterMs ? 1 : 0, { server });
out.add("homelab_server_last_seen_timestamp_seconds", "gauge", "When the server's agent last reported, as a Unix time.", Math.floor(seen / 1000), { server });
if (s.cpuLoadPercent !== null) out.add("homelab_server_cpu_load_percent", "gauge", "CPU load as the agent reports it.", s.cpuLoadPercent, { server });
if (s.memTotalBytes && s.memUsedBytes !== null) out.add("homelab_server_memory_used_percent", "gauge", "Memory in use.", (s.memUsedBytes / s.memTotalBytes) * 100, { server });
let disks: { mount: string; sizeBytes: number; usedBytes: number }[] = [];
try {
disks = s.disks ? JSON.parse(s.disks) : [];
} catch {
// a garbled report shouldn't blank the whole scrape
}
for (const d of disks) {
if (d.sizeBytes > 0) out.add("homelab_server_disk_used_percent", "gauge", "How full a disk is.", (d.usedBytes / d.sizeBytes) * 100, { server, mount: d.mount });
}
const updates = parsePendingUpdates(s.pendingUpdates);
if (updates) {
out.add("homelab_server_pending_updates", "gauge", "Updates the agent says are waiting to be installed.", updates.count, { server });
if (updates.security !== null) out.add("homelab_server_pending_security_updates", "gauge", "Of those, the ones that are security fixes.", updates.security, { server });
out.add("homelab_server_reboot_required", "gauge", "1 if a restart is due.", updates.rebootRequired ? 1 : 0, { server });
}
out.add("homelab_server_agent_outdated", "gauge", "1 if the server's agent is older than the one this installation would install now.", isAgentOutdated(s, latest) ? 1 : 0, { server });
}
try {
const uptime = await getUptime(now, 30);
for (const s of serverRows) {
const series = uptime.servers[s.id];
if (series) out.add("homelab_server_uptime_ratio_30d", "gauge", "Share of the last 30 days a server was up, 0 to 1.", series.percent / 100, { server: s.name });
}
} catch (err) {
console.error("[metrics] couldn't read the uptime:", err);
}
// ── Alerts ──
try {
const report = await getAlerts(false);
for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity });
out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced);
out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped);
} catch (err) {
console.error("[metrics] couldn't read the alerts:", err);
}
// ── Integrations ──
for (const i of await db.select({ name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations)) {
if (!i.enabled) continue;
const status = getSourceStatus(i.type).status;
if (status === "unknown") continue; // nothing has called it yet: not up, not down
out.add("homelab_integration_up", "gauge", "1 if the integration's latest calls are working, 0 if they are failing.", status === "ok" ? 1 : 0, { integration: i.name, type: i.type });
}
// ── Maintenance ──
const iso = now.toISOString();
const active = await db.select({ id: maintenanceWindows.id }).from(maintenanceWindows).where(and(lte(maintenanceWindows.startedAt, iso), gt(maintenanceWindows.endsAt, iso)));
out.add("homelab_maintenance_active_windows", "gauge", "Maintenance windows running now.", active.length);
// ── Things that expire ──
for (const s of await db.select().from(secrets)) {
const status = computeSecretStatus(s.expiryDate, s.warnDays, now);
if (status.daysLeft !== null) out.add("homelab_secret_expiry_days", "gauge", "Days until a secret expires (negative once it has).", status.daysLeft, { name: s.name });
}
// ── Inventory ──
const power = (await db.select({ w: assets.powerWatts }).from(assets)).reduce((n, a) => n + (a.w ?? 0), 0);
out.add("homelab_inventory_power_watts", "gauge", "Typical power draw of everything in the inventory, in watts.", power);
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
return out.toString();
}
+37
View File
@@ -0,0 +1,37 @@
/**
* Metrics tokens: read-only access to /metrics for Prometheus. Like an agent token or a share link, the secret is shown once, when it is
* made, and only its SHA-256 is kept — so it can't be read back out of the database or a backup.
*/
import { createHash, randomBytes } from "node:crypto";
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { metricsTokens } from "../db/schema.js";
export type MetricsTokenRow = typeof metricsTokens.$inferSelect;
const hash = (token: string) => createHash("sha256").update(token).digest("hex");
/** Use is not recorded more than once in this time, so a scrape every fifteen seconds doesn't write the database each time. */
const USED_WRITE_GAP_MS = 60_000;
export async function createMetricsToken(params: { label: string; createdBy: string }): Promise<{ row: MetricsTokenRow; token: string }> {
const token = `hlm_m_${randomBytes(24).toString("base64url")}`;
const [row] = await db.insert(metricsTokens).values({ label: params.label, tokenHash: hash(token), tokenPrefix: token.slice(0, 12), createdBy: params.createdBy }).returning();
return { row, token };
}
/** The token a secret belongs to, if there is one; notes that it was used. */
export async function resolveMetricsToken(token: string, now: Date = new Date()): Promise<MetricsTokenRow | null> {
if (typeof token !== "string" || token.length < 20 || token.length > 128) return null;
const [row] = await db.select().from(metricsTokens).where(eq(metricsTokens.tokenHash, hash(token))).limit(1);
if (!row) return null;
if (!row.lastUsedAt || now.getTime() - Date.parse(row.lastUsedAt) >= USED_WRITE_GAP_MS) {
await db.update(metricsTokens).set({ lastUsedAt: now.toISOString() }).where(eq(metricsTokens.id, row.id));
}
return row;
}
/** A token as the admin list shows it: everything but the hash. */
export function publicMetricsToken(row: MetricsTokenRow) {
return { id: row.id, label: row.label, tokenPrefix: row.tokenPrefix, createdBy: row.createdBy, createdAt: row.createdAt, lastUsedAt: row.lastUsedAt };
}
+13 -4
View File
@@ -1,6 +1,7 @@
import nodemailer from "nodemailer";
import { getSettings, type NotificationEvents } from "./settingsStore.js";
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
import { emitEvent } from "./eventWebhooks.js";
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
@@ -309,7 +310,9 @@ export async function notifyPbsVerificationFailed(
);
}
export async function notifyHealthIssues(issues: { message: string }[]): Promise<void> {
export async function notifyHealthIssues(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
if (!(await eventEnabled("healthAlerts"))) return;
await notify(
@@ -318,7 +321,8 @@ export async function notifyHealthIssues(issues: { message: string }[]): Promise
);
}
export async function notifyHealthRecovered(resolved: { message: string }[]): Promise<void> {
export async function notifyHealthRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
if (!(await eventEnabled("healthAlerts"))) return;
await notify(
@@ -327,7 +331,9 @@ export async function notifyHealthRecovered(resolved: { message: string }[]): Pr
);
}
export async function notifyAutomationFailed(issues: { message: string }[]): Promise<void> {
export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
if (issues.length === 0) return;
if (!(await eventEnabled("automationAlerts"))) return;
await notify(
@@ -336,7 +342,8 @@ export async function notifyAutomationFailed(issues: { message: string }[]): Pro
);
}
export async function notifyAutomationRecovered(resolved: { message: string }[]): Promise<void> {
export async function notifyAutomationRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
if (resolved.length === 0) return;
if (!(await eventEnabled("automationAlerts"))) return;
await notify(
@@ -409,6 +416,7 @@ export function sourceLabel(source: string): string {
}
export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise<void> {
emitEvent("alert.opened", { alert: { key: `integration:${source}`, source, severity: "warning", message: `${sourceLabel(source)} has failed its last ${consecutiveFailures} calls in a row.` } });
if (!(await eventEnabled("integrationFailureAlerts"))) return;
await notify(
"Homelab Manager — Integration Down",
@@ -417,6 +425,7 @@ export async function notifyIntegrationDown(source: string, consecutiveFailures:
}
export async function notifyIntegrationRecovered(source: string): Promise<void> {
emitEvent("alert.resolved", { alert: { key: `integration:${source}`, message: `${sourceLabel(source)} is answering again.` } });
if (!(await eventEnabled("integrationFailureAlerts"))) return;
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
}
+22
View File
@@ -108,6 +108,24 @@ export interface DigestSettings {
time: string;
}
/** How roles are decided at sign-in: by hand on the Users page (the default), or from the groups the identity provider reports. */
export interface SignInSettings {
roleFromGroups: boolean;
/** The claim that carries the groups; "groups" in Authentik. */
groupsClaim: string;
adminGroups: string[];
operatorGroups: string[];
/** Someone in none of the named groups gets viewer, or keeps what they have. */
otherwise: "viewer" | "keep";
}
/** What the inventory needs to work out a running cost. Without a price per kWh it shows draw but no cost. */
export interface InventorySettings {
/** A short name shown after amounts ("SEK", "kr", "$"); may be blank. */
currency: string;
pricePerKwh: number | null;
}
export interface ConsistencySettings {
/** CIDR ranges and single addresses the consistency report ignores entirely — e.g. Docker networks that repeat on many hosts. */
excludedRanges: string[];
@@ -160,6 +178,8 @@ export interface AppSettings {
quietHours: QuietHoursSettings;
healthChecks: HealthCheckSettings;
digest: DigestSettings;
signIn: SignInSettings;
inventory: InventorySettings;
consistency: ConsistencySettings;
nameGenerator: NameGeneratorSettings;
alertTickets: AlertTicketSettings;
@@ -193,6 +213,8 @@ const DEFAULTS: AppSettings = {
logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 },
quietHours: { enabled: false, start: "22:00", end: "07:00" },
digest: { enabled: false, day: 1, time: "08:00" },
inventory: { currency: "", pricePerKwh: null },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
+26 -1
View File
@@ -5,9 +5,11 @@
* into a few lines per topic and leaves out any topic with nothing to say. Plain text, so it reads in a phone notification and an email alike.
*/
import { db } from "../db/client.js";
import { integrations, secrets, servers } from "../db/schema.js";
import { assets, integrations, secrets, servers } from "../db/schema.js";
import { warrantyDaysLeft } from "./inventory.js";
import { env } from "../env.js";
import { getAlerts } from "./alerts.js";
import { getBackupCoverage } from "./backupCoverage.js";
import { getChangesSince } from "./changesSince.js";
import { collectDomainAlerts } from "./domainMonitor.js";
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
@@ -157,6 +159,13 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
for (const d of (await collectDomainAlerts(now)).expiring) {
expiring.push({ daysLeft: d.daysLeft, text: d.daysLeft < 0 ? `Domain ${d.name} expired ${plural(Math.abs(d.daysLeft), "day")} ago` : `Domain ${d.name} expires in ${plural(d.daysLeft, "day")} (${d.expiresAt})` });
}
// A warranty about to end is worth knowing while there is still time to claim on it; one that ended long ago is not news.
for (const a of await db.select({ name: assets.name, warrantyUntil: assets.warrantyUntil }).from(assets)) {
const days = warrantyDaysLeft(a.warrantyUntil, now);
if (days !== null && days >= -30 && days <= 60) {
expiring.push({ daysLeft: days, text: days < 0 ? `Warranty on ${a.name} ended ${plural(Math.abs(days), "day")} ago` : `Warranty on ${a.name} ends in ${plural(days, "day")} (${a.warrantyUntil})` });
}
}
if (expiring.length > 0) {
expiring.sort((a, b) => a.daysLeft - b.daysLeft);
sections.push({ heading: "Expiring", lines: listWithMore(expiring.map((e) => `- ${e.text}`)), attention: true });
@@ -185,6 +194,22 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
console.error("[digest] couldn't read the pending updates:", err);
}
// ── Backups ──
try {
const coverage = await getBackupCoverage({ now });
const order = { failed: 0, none: 1, stale: 2 } as const;
const bad = coverage.guests.filter((g): g is typeof g & { status: keyof typeof order } => g.status === "failed" || g.status === "none" || g.status === "stale").sort((a, b) => order[a.status] - order[b.status] || a.name.localeCompare(b.name));
if (bad.length > 0) {
const what = (g: (typeof bad)[number]) =>
g.status === "failed" ? `${g.failure}` : g.status === "none" ? "no backup, and not in any backup job" : `last backup ${spanWords((now.getTime() - Date.parse(g.lastBackupAt!)) / 60_000)} ago`;
const lines = [`${coverage.counts.ok} of ${plural(coverage.counts.total, "guest")} ${coverage.counts.ok === 1 ? "has" : "have"} a recent backup.`];
lines.push(...listWithMore(bad.map((g) => `- ${g.name} (${g.type === "qemu" ? "VM" : "container"} ${g.vmid}): ${what(g)}`)));
sections.push({ heading: "Backups", lines, attention: true });
}
} catch (err) {
console.error("[digest] couldn't read the backup coverage:", err);
}
// ── Planned ──
try {
const horizon = now.getTime() + DIGEST_DAYS * DAY_MS;