Service-level targets with error budgets, and quarterly and annual reports

Monitoring -> Service levels: a target (99.9 %) on an endpoint check, a server or an
integration, measured over calendar months. Shows the figure, downtime used, the
error budget left, and how the last six months went. 80 % of the budget used is a
warning and an overspent budget a critical alert. Endpoints are measured day by day,
servers and integrations to the minute from the audit log.

The report page gains Quarter and Year (same figures, with a Service levels section),
and two settings send the quarterly and annual report after the month that ends them.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-21 22:20:07 +02:00
1 parent 31044b2b64
commit 3b31b16796
31 files changed
+7706 -61

No files matched your search

+13
View File
@@ -7,6 +7,7 @@ import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { latestTests } from "./restoreTests.js";
import { heartbeatState, parseStamp } from "./heartbeats.js";
import { env } from "../env.js";
@@ -58,6 +59,17 @@ class Exposition {
}
}
/** Each service-level target, for /metrics: its target, how it is doing this month and how much error budget is left. */
async function addServiceLevelMetrics(out: Exposition, now: Date) {
for (const s of await levelStatuses(now, { history: false })) {
const labels = { name: s.name, subject: s.subjectName ?? "", type: s.subjectType };
out.add("homelab_service_level_target_percent", "gauge", "The availability a service-level target asks for.", s.target, labels);
if (s.measure.percent === null) continue;
out.add("homelab_service_level_availability_percent", "gauge", "How available the thing has been so far this calendar month.", s.measure.percent, labels);
out.add("homelab_service_level_error_budget_remaining_minutes", "gauge", "Downtime still allowed this month before the target is missed (negative once it is).", Math.round(s.budget.remainingMinutes * 10) / 10, labels);
}
}
/** Each UPS, for /metrics: whether it is on battery, how full the battery is, how long it would last and how loaded it is. */
async function addUpsMetrics(out: Exposition, now: Date) {
const fresh = new Set<number>();
@@ -239,6 +251,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
await addEndpointMetrics(out, now);
await addUpsMetrics(out, now);
await addServiceLevelMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}