Service-level targets with error budgets, and quarterly and annual reports

Monitoring -> Service levels: a target (99.9 %) on an endpoint check, a server or an
integration, measured over calendar months. Shows the figure, downtime used, the
error budget left, and how the last six months went. 80 % of the budget used is a
warning and an overspent budget a critical alert. Endpoints are measured day by day,
servers and integrations to the minute from the audit log.

The report page gains Quarter and Year (same figures, with a Service levels section),
and two settings send the quarterly and annual report after the month that ends them.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-21 22:20:07 +02:00
1 parent 31044b2b64
commit 3b31b16796
31 files changed
+7706 -61

No files matched your search

+78 -9
View File
@@ -34,13 +34,17 @@ export interface UptimeSeries {
percent: number;
}
export interface Segment {
from: number;
to: number;
down: boolean;
}
/**
* Splits [knownFrom, now] into up and down stretches from the recorded transitions and lays them over `days` consecutive 24-hour
* buckets ending at `now`. Before the first transition the thing is taken to have been in the opposite state (a "went offline" means it
* was up until then); with no transitions at all it has been up the whole time.
* Splits [knownFrom, now] into up and down stretches from the recorded transitions. Before the first transition the thing is taken to have been in the opposite state
* (a "went offline" means it was up until then); with no transitions at all it has been up the whole time.
*/
export function computeBuckets(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeBucket[] {
const start = now - days * DAY_MS;
export function segmentsOf(transitions: Transition[], knownFrom: number, now: number): Segment[] {
const sorted = [...transitions].sort((a, b) => a.at - b.at);
// State at knownFrom: after any transitions that happened before it, otherwise the opposite of the first one that follows.
@@ -49,7 +53,7 @@ export function computeBuckets(transitions: Transition[], knownFrom: number, now
while (i < sorted.length && sorted[i].at <= knownFrom) state = sorted[i++].down;
if (i === 0 && sorted.length > 0) state = !sorted[0].down;
const segments: { from: number; to: number; down: boolean }[] = [];
const segments: Segment[] = [];
let cursor = Math.min(knownFrom, now);
for (; i < sorted.length; i++) {
const t = Math.min(sorted[i].at, now);
@@ -58,6 +62,26 @@ export function computeBuckets(transitions: Transition[], knownFrom: number, now
state = sorted[i].down;
}
if (now > cursor) segments.push({ from: cursor, to: now, down: state });
return segments;
}
/** Minutes known, and of those spent down, within [from, to]. */
export function minutesWithin(segments: Segment[], from: number, to: number): { known: number; down: number } {
let known = 0;
let down = 0;
for (const seg of segments) {
const overlap = Math.min(seg.to, to) - Math.max(seg.from, from);
if (overlap <= 0) continue;
known += overlap;
if (seg.down) down += overlap;
}
return { known: Math.round(known / 60_000), down: Math.round(down / 60_000) };
}
/** The segments laid over `days` consecutive 24-hour buckets ending at `now`. */
export function computeBuckets(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeBucket[] {
const start = now - days * DAY_MS;
const segments = segmentsOf(transitions, knownFrom, now);
const known = new Array<number>(days).fill(0);
const down = new Array<number>(days).fill(0);
@@ -121,10 +145,49 @@ export interface UptimeReport {
integrations: Record<number, UptimeSeries>;
}
export async function getUptime(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<UptimeReport> {
const nowMs = now.getTime();
const rows = await db.select().from(auditLog).where(inArray(auditLog.action, ["went_offline", "back_online", "went_failing", "recovered"]));
/** How long a server or integration was known, and down, between two moments. */
export interface UptimeTotals {
known: number;
down: number;
}
export interface UptimeRange {
servers: Record<number, UptimeTotals>;
integrations: Record<number, UptimeTotals>;
}
/**
* Exactly how long each server and integration was down between `from` and `to` (the end is clipped to now), to the minute — for a target that must be measured over a
* calendar period rather than in whole days back from now. Only the ones known for some of it are listed.
*/
export async function getUptimeRange(from: Date, to: Date, now: Date = new Date()): Promise<UptimeRange> {
const nowMs = now.getTime();
const end = Math.min(to.getTime(), nowMs);
const start = from.getTime();
const out: UptimeRange = { servers: {}, integrations: {} };
if (end <= start) return out;
const { serverEvents, sourceEvents } = await loadTransitions(nowMs);
for (const s of await db.select({ id: servers.id, createdAt: servers.createdAt, lastSeenAt: servers.lastSeenAt }).from(servers)) {
const events = serverEvents.get(s.id) ?? [];
if (!s.lastSeenAt && events.length === 0) continue;
const created = Date.parse(auditTimeToIso(s.createdAt));
const totals = minutesWithin(segmentsOf(events, Number.isFinite(created) ? created : nowMs, nowMs), start, end);
if (totals.known > 0) out.servers[s.id] = totals;
}
const since = await ensureHistoryStart(nowMs);
for (const i of await db.select({ id: integrations.id, type: integrations.type, createdAt: integrations.createdAt }).from(integrations)) {
const created = Date.parse(auditTimeToIso(i.createdAt));
const knownFrom = Math.max(Number.isFinite(created) ? created : 0, since);
const totals = minutesWithin(segmentsOf(sourceEvents.get(i.type) ?? [], knownFrom, nowMs), start, end);
if (totals.known > 0) out.integrations[i.id] = totals;
}
return out;
}
/** The recorded moments servers went away and came back, and integrations started failing and worked again. */
async function loadTransitions(nowMs: number): Promise<{ serverEvents: Map<number, Transition[]>; sourceEvents: Map<string, Transition[]> }> {
const rows = await db.select().from(auditLog).where(inArray(auditLog.action, ["went_offline", "back_online", "went_failing", "recovered"]));
const serverEvents = new Map<number, Transition[]>();
const sourceEvents = new Map<string, Transition[]>();
for (const r of rows) {
@@ -141,6 +204,12 @@ export async function getUptime(now: Date = new Date(), days: number = DEFAULT_D
sourceEvents.set(r.targetId, list);
}
}
return { serverEvents, sourceEvents };
}
export async function getUptime(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<UptimeReport> {
const nowMs = now.getTime();
const { serverEvents, sourceEvents } = await loadTransitions(nowMs);
const out: UptimeReport = { days, endsAt: now.toISOString(), servers: {}, integrations: {} };