Service-level targets with error budgets, and quarterly and annual reports
Monitoring -> Service levels: a target (99.9 %) on an endpoint check, a server or an integration, measured over calendar months. Shows the figure, downtime used, the error budget left, and how the last six months went. 80 % of the budget used is a warning and an overspent budget a critical alert. Endpoints are measured day by day, servers and integrations to the minute from the audit log. The report page gains Quarter and Year (same figures, with a Service levels section), and two settings send the quarterly and annual report after the month that ends them. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
31044b2b64
commit
3b31b16796
31 files changed
+7706
-61
No files matched your search
@@ -7,6 +7,7 @@ import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { heartbeatState, parseStamp } from "./heartbeats.js";
|
||||
import { env } from "../env.js";
|
||||
@@ -58,6 +59,17 @@ class Exposition {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each service-level target, for /metrics: its target, how it is doing this month and how much error budget is left. */
|
||||
async function addServiceLevelMetrics(out: Exposition, now: Date) {
|
||||
for (const s of await levelStatuses(now, { history: false })) {
|
||||
const labels = { name: s.name, subject: s.subjectName ?? "", type: s.subjectType };
|
||||
out.add("homelab_service_level_target_percent", "gauge", "The availability a service-level target asks for.", s.target, labels);
|
||||
if (s.measure.percent === null) continue;
|
||||
out.add("homelab_service_level_availability_percent", "gauge", "How available the thing has been so far this calendar month.", s.measure.percent, labels);
|
||||
out.add("homelab_service_level_error_budget_remaining_minutes", "gauge", "Downtime still allowed this month before the target is missed (negative once it is).", Math.round(s.budget.remainingMinutes * 10) / 10, labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Each UPS, for /metrics: whether it is on battery, how full the battery is, how long it would last and how loaded it is. */
|
||||
async function addUpsMetrics(out: Exposition, now: Date) {
|
||||
const fresh = new Set<number>();
|
||||
@@ -239,6 +251,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
|
||||
await addEndpointMetrics(out, now);
|
||||
await addUpsMetrics(out, now);
|
||||
await addServiceLevelMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
Reference in new issue
Block a user