Service-level targets with error budgets, and quarterly and annual reports
Monitoring -> Service levels: a target (99.9 %) on an endpoint check, a server or an integration, measured over calendar months. Shows the figure, downtime used, the error budget left, and how the last six months went. 80 % of the budget used is a warning and an overspent budget a critical alert. Endpoints are measured day by day, servers and integrations to the minute from the audit log. The report page gains Quarter and Year (same figures, with a Service levels section), and two settings send the quarterly and annual report after the month that ends them. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
31044b2b64
commit
3b31b16796
31 files changed
+7706
-61
No files matched your search
@@ -34,13 +34,17 @@ export interface UptimeSeries {
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface Segment {
|
||||
from: number;
|
||||
to: number;
|
||||
down: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits [knownFrom, now] into up and down stretches from the recorded transitions and lays them over `days` consecutive 24-hour
|
||||
* buckets ending at `now`. Before the first transition the thing is taken to have been in the opposite state (a "went offline" means it
|
||||
* was up until then); with no transitions at all it has been up the whole time.
|
||||
* Splits [knownFrom, now] into up and down stretches from the recorded transitions. Before the first transition the thing is taken to have been in the opposite state
|
||||
* (a "went offline" means it was up until then); with no transitions at all it has been up the whole time.
|
||||
*/
|
||||
export function computeBuckets(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeBucket[] {
|
||||
const start = now - days * DAY_MS;
|
||||
export function segmentsOf(transitions: Transition[], knownFrom: number, now: number): Segment[] {
|
||||
const sorted = [...transitions].sort((a, b) => a.at - b.at);
|
||||
|
||||
// State at knownFrom: after any transitions that happened before it, otherwise the opposite of the first one that follows.
|
||||
@@ -49,7 +53,7 @@ export function computeBuckets(transitions: Transition[], knownFrom: number, now
|
||||
while (i < sorted.length && sorted[i].at <= knownFrom) state = sorted[i++].down;
|
||||
if (i === 0 && sorted.length > 0) state = !sorted[0].down;
|
||||
|
||||
const segments: { from: number; to: number; down: boolean }[] = [];
|
||||
const segments: Segment[] = [];
|
||||
let cursor = Math.min(knownFrom, now);
|
||||
for (; i < sorted.length; i++) {
|
||||
const t = Math.min(sorted[i].at, now);
|
||||
@@ -58,6 +62,26 @@ export function computeBuckets(transitions: Transition[], knownFrom: number, now
|
||||
state = sorted[i].down;
|
||||
}
|
||||
if (now > cursor) segments.push({ from: cursor, to: now, down: state });
|
||||
return segments;
|
||||
}
|
||||
|
||||
/** Minutes known, and of those spent down, within [from, to]. */
|
||||
export function minutesWithin(segments: Segment[], from: number, to: number): { known: number; down: number } {
|
||||
let known = 0;
|
||||
let down = 0;
|
||||
for (const seg of segments) {
|
||||
const overlap = Math.min(seg.to, to) - Math.max(seg.from, from);
|
||||
if (overlap <= 0) continue;
|
||||
known += overlap;
|
||||
if (seg.down) down += overlap;
|
||||
}
|
||||
return { known: Math.round(known / 60_000), down: Math.round(down / 60_000) };
|
||||
}
|
||||
|
||||
/** The segments laid over `days` consecutive 24-hour buckets ending at `now`. */
|
||||
export function computeBuckets(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeBucket[] {
|
||||
const start = now - days * DAY_MS;
|
||||
const segments = segmentsOf(transitions, knownFrom, now);
|
||||
|
||||
const known = new Array<number>(days).fill(0);
|
||||
const down = new Array<number>(days).fill(0);
|
||||
@@ -121,10 +145,49 @@ export interface UptimeReport {
|
||||
integrations: Record<number, UptimeSeries>;
|
||||
}
|
||||
|
||||
export async function getUptime(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<UptimeReport> {
|
||||
const nowMs = now.getTime();
|
||||
const rows = await db.select().from(auditLog).where(inArray(auditLog.action, ["went_offline", "back_online", "went_failing", "recovered"]));
|
||||
/** How long a server or integration was known, and down, between two moments. */
|
||||
export interface UptimeTotals {
|
||||
known: number;
|
||||
down: number;
|
||||
}
|
||||
|
||||
export interface UptimeRange {
|
||||
servers: Record<number, UptimeTotals>;
|
||||
integrations: Record<number, UptimeTotals>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Exactly how long each server and integration was down between `from` and `to` (the end is clipped to now), to the minute — for a target that must be measured over a
|
||||
* calendar period rather than in whole days back from now. Only the ones known for some of it are listed.
|
||||
*/
|
||||
export async function getUptimeRange(from: Date, to: Date, now: Date = new Date()): Promise<UptimeRange> {
|
||||
const nowMs = now.getTime();
|
||||
const end = Math.min(to.getTime(), nowMs);
|
||||
const start = from.getTime();
|
||||
const out: UptimeRange = { servers: {}, integrations: {} };
|
||||
if (end <= start) return out;
|
||||
const { serverEvents, sourceEvents } = await loadTransitions(nowMs);
|
||||
|
||||
for (const s of await db.select({ id: servers.id, createdAt: servers.createdAt, lastSeenAt: servers.lastSeenAt }).from(servers)) {
|
||||
const events = serverEvents.get(s.id) ?? [];
|
||||
if (!s.lastSeenAt && events.length === 0) continue;
|
||||
const created = Date.parse(auditTimeToIso(s.createdAt));
|
||||
const totals = minutesWithin(segmentsOf(events, Number.isFinite(created) ? created : nowMs, nowMs), start, end);
|
||||
if (totals.known > 0) out.servers[s.id] = totals;
|
||||
}
|
||||
const since = await ensureHistoryStart(nowMs);
|
||||
for (const i of await db.select({ id: integrations.id, type: integrations.type, createdAt: integrations.createdAt }).from(integrations)) {
|
||||
const created = Date.parse(auditTimeToIso(i.createdAt));
|
||||
const knownFrom = Math.max(Number.isFinite(created) ? created : 0, since);
|
||||
const totals = minutesWithin(segmentsOf(sourceEvents.get(i.type) ?? [], knownFrom, nowMs), start, end);
|
||||
if (totals.known > 0) out.integrations[i.id] = totals;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The recorded moments servers went away and came back, and integrations started failing and worked again. */
|
||||
async function loadTransitions(nowMs: number): Promise<{ serverEvents: Map<number, Transition[]>; sourceEvents: Map<string, Transition[]> }> {
|
||||
const rows = await db.select().from(auditLog).where(inArray(auditLog.action, ["went_offline", "back_online", "went_failing", "recovered"]));
|
||||
const serverEvents = new Map<number, Transition[]>();
|
||||
const sourceEvents = new Map<string, Transition[]>();
|
||||
for (const r of rows) {
|
||||
@@ -141,6 +204,12 @@ export async function getUptime(now: Date = new Date(), days: number = DEFAULT_D
|
||||
sourceEvents.set(r.targetId, list);
|
||||
}
|
||||
}
|
||||
return { serverEvents, sourceEvents };
|
||||
}
|
||||
|
||||
export async function getUptime(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<UptimeReport> {
|
||||
const nowMs = now.getTime();
|
||||
const { serverEvents, sourceEvents } = await loadTransitions(nowMs);
|
||||
|
||||
const out: UptimeReport = { days, endsAt: now.toISOString(), servers: {}, integrations: {} };
|
||||
|
||||
|
||||
Reference in new issue
Block a user