Monitoring -> Service levels: a target (99.9 %) on an endpoint check, a server or an integration, measured over calendar months. Shows the figure, downtime used, the error budget left, and how the last six months went. 80 % of the budget used is a warning and an overspent budget a critical alert. Endpoints are measured day by day, servers and integrations to the minute from the audit log. The report page gains Quarter and Year (same figures, with a Service levels section), and two settings send the quarterly and annual report after the month that ends them. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
234 lines
9.9 KiB
TypeScript
234 lines
9.9 KiB
TypeScript
/**
|
|
* How each server and integration has done over the last weeks, as one figure per day: how long it was up, how long down, and for
|
|
* how much of the day anything is known. It is worked out from the moments the health checks wrote down in the audit log (a
|
|
* server stopped reporting / reported again; an integration's calls started failing / worked again), so it needs no table of its
|
|
* own — and it can only be as old as those records.
|
|
*/
|
|
import { inArray } from "drizzle-orm";
|
|
import { db } from "../db/client.js";
|
|
import { auditLog, integrations, servers } from "../db/schema.js";
|
|
import { auditTimeToIso } from "./serverActivity.js";
|
|
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
|
|
|
const DAY_MS = 24 * 3600_000;
|
|
const HISTORY_FLAG = "integration_history_since";
|
|
|
|
export const DEFAULT_DAYS = 30;
|
|
|
|
export interface Transition {
|
|
/** Epoch ms. */
|
|
at: number;
|
|
down: boolean;
|
|
}
|
|
|
|
/** Minutes of one 24-hour stretch that are known, and of those, minutes spent down. */
|
|
export interface UptimeBucket {
|
|
k: number;
|
|
d: number;
|
|
}
|
|
|
|
export interface UptimeSeries {
|
|
/** Oldest first; the last bucket ends now. */
|
|
buckets: UptimeBucket[];
|
|
/** Share of the known time spent up over the whole period. */
|
|
percent: number;
|
|
}
|
|
|
|
export interface Segment {
|
|
from: number;
|
|
to: number;
|
|
down: boolean;
|
|
}
|
|
|
|
/**
|
|
* Splits [knownFrom, now] into up and down stretches from the recorded transitions. Before the first transition the thing is taken to have been in the opposite state
|
|
* (a "went offline" means it was up until then); with no transitions at all it has been up the whole time.
|
|
*/
|
|
export function segmentsOf(transitions: Transition[], knownFrom: number, now: number): Segment[] {
|
|
const sorted = [...transitions].sort((a, b) => a.at - b.at);
|
|
|
|
// State at knownFrom: after any transitions that happened before it, otherwise the opposite of the first one that follows.
|
|
let state = false;
|
|
let i = 0;
|
|
while (i < sorted.length && sorted[i].at <= knownFrom) state = sorted[i++].down;
|
|
if (i === 0 && sorted.length > 0) state = !sorted[0].down;
|
|
|
|
const segments: Segment[] = [];
|
|
let cursor = Math.min(knownFrom, now);
|
|
for (; i < sorted.length; i++) {
|
|
const t = Math.min(sorted[i].at, now);
|
|
if (t > cursor) segments.push({ from: cursor, to: t, down: state });
|
|
cursor = Math.max(cursor, t);
|
|
state = sorted[i].down;
|
|
}
|
|
if (now > cursor) segments.push({ from: cursor, to: now, down: state });
|
|
return segments;
|
|
}
|
|
|
|
/** Minutes known, and of those spent down, within [from, to]. */
|
|
export function minutesWithin(segments: Segment[], from: number, to: number): { known: number; down: number } {
|
|
let known = 0;
|
|
let down = 0;
|
|
for (const seg of segments) {
|
|
const overlap = Math.min(seg.to, to) - Math.max(seg.from, from);
|
|
if (overlap <= 0) continue;
|
|
known += overlap;
|
|
if (seg.down) down += overlap;
|
|
}
|
|
return { known: Math.round(known / 60_000), down: Math.round(down / 60_000) };
|
|
}
|
|
|
|
/** The segments laid over `days` consecutive 24-hour buckets ending at `now`. */
|
|
export function computeBuckets(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeBucket[] {
|
|
const start = now - days * DAY_MS;
|
|
const segments = segmentsOf(transitions, knownFrom, now);
|
|
|
|
const known = new Array<number>(days).fill(0);
|
|
const down = new Array<number>(days).fill(0);
|
|
for (const seg of segments) {
|
|
if (seg.to <= start) continue;
|
|
const first = Math.max(0, Math.floor((Math.max(seg.from, start) - start) / DAY_MS));
|
|
for (let b = first; b < days; b++) {
|
|
const bStart = start + b * DAY_MS;
|
|
if (bStart >= seg.to) break;
|
|
const overlap = Math.min(seg.to, bStart + DAY_MS) - Math.max(seg.from, bStart);
|
|
if (overlap <= 0) continue;
|
|
known[b] += overlap;
|
|
if (seg.down) down[b] += overlap;
|
|
}
|
|
}
|
|
return known.map((k, b) => ({ k: Math.round(k / 60_000), d: Math.round(down[b] / 60_000) }));
|
|
}
|
|
|
|
/** Share of the known time spent up, to one decimal; null when nothing is known. */
|
|
export function uptimePercent(buckets: UptimeBucket[]): number | null {
|
|
const known = buckets.reduce((n, b) => n + b.k, 0);
|
|
if (known === 0) return null;
|
|
const down = buckets.reduce((n, b) => n + b.d, 0);
|
|
return Math.round((1 - down / known) * 1000) / 10;
|
|
}
|
|
|
|
function series(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeSeries | null {
|
|
const buckets = computeBuckets(transitions, knownFrom, now, days);
|
|
const percent = uptimePercent(buckets);
|
|
return percent === null ? null : { buckets, percent };
|
|
}
|
|
|
|
/**
|
|
* Integrations' failing / working-again moments only exist from the version that began writing them, so nothing earlier can be claimed.
|
|
* The first time anything asks, that moment is written down and kept.
|
|
*/
|
|
let historyStart: number | null = null;
|
|
export async function ensureHistoryStart(now: number = Date.now()): Promise<number> {
|
|
if (historyStart !== null) return historyStart;
|
|
const stored = await getInternalFlag(HISTORY_FLAG);
|
|
const parsed = stored ? Date.parse(stored) : NaN;
|
|
if (Number.isFinite(parsed)) {
|
|
historyStart = parsed;
|
|
return parsed;
|
|
}
|
|
await setInternalFlag(HISTORY_FLAG, new Date(now).toISOString());
|
|
historyStart = now;
|
|
return now;
|
|
}
|
|
|
|
/** For tests: forget the remembered start. */
|
|
export function resetHistoryStartForTests(): void {
|
|
historyStart = null;
|
|
}
|
|
|
|
export interface UptimeReport {
|
|
days: number;
|
|
/** ISO — the end of the last bucket. */
|
|
endsAt: string;
|
|
servers: Record<number, UptimeSeries>;
|
|
integrations: Record<number, UptimeSeries>;
|
|
}
|
|
|
|
/** How long a server or integration was known, and down, between two moments. */
|
|
export interface UptimeTotals {
|
|
known: number;
|
|
down: number;
|
|
}
|
|
|
|
export interface UptimeRange {
|
|
servers: Record<number, UptimeTotals>;
|
|
integrations: Record<number, UptimeTotals>;
|
|
}
|
|
|
|
/**
|
|
* Exactly how long each server and integration was down between `from` and `to` (the end is clipped to now), to the minute — for a target that must be measured over a
|
|
* calendar period rather than in whole days back from now. Only the ones known for some of it are listed.
|
|
*/
|
|
export async function getUptimeRange(from: Date, to: Date, now: Date = new Date()): Promise<UptimeRange> {
|
|
const nowMs = now.getTime();
|
|
const end = Math.min(to.getTime(), nowMs);
|
|
const start = from.getTime();
|
|
const out: UptimeRange = { servers: {}, integrations: {} };
|
|
if (end <= start) return out;
|
|
const { serverEvents, sourceEvents } = await loadTransitions(nowMs);
|
|
|
|
for (const s of await db.select({ id: servers.id, createdAt: servers.createdAt, lastSeenAt: servers.lastSeenAt }).from(servers)) {
|
|
const events = serverEvents.get(s.id) ?? [];
|
|
if (!s.lastSeenAt && events.length === 0) continue;
|
|
const created = Date.parse(auditTimeToIso(s.createdAt));
|
|
const totals = minutesWithin(segmentsOf(events, Number.isFinite(created) ? created : nowMs, nowMs), start, end);
|
|
if (totals.known > 0) out.servers[s.id] = totals;
|
|
}
|
|
const since = await ensureHistoryStart(nowMs);
|
|
for (const i of await db.select({ id: integrations.id, type: integrations.type, createdAt: integrations.createdAt }).from(integrations)) {
|
|
const created = Date.parse(auditTimeToIso(i.createdAt));
|
|
const knownFrom = Math.max(Number.isFinite(created) ? created : 0, since);
|
|
const totals = minutesWithin(segmentsOf(sourceEvents.get(i.type) ?? [], knownFrom, nowMs), start, end);
|
|
if (totals.known > 0) out.integrations[i.id] = totals;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/** The recorded moments servers went away and came back, and integrations started failing and worked again. */
|
|
async function loadTransitions(nowMs: number): Promise<{ serverEvents: Map<number, Transition[]>; sourceEvents: Map<string, Transition[]> }> {
|
|
const rows = await db.select().from(auditLog).where(inArray(auditLog.action, ["went_offline", "back_online", "went_failing", "recovered"]));
|
|
const serverEvents = new Map<number, Transition[]>();
|
|
const sourceEvents = new Map<string, Transition[]>();
|
|
for (const r of rows) {
|
|
const at = Date.parse(auditTimeToIso(r.createdAt));
|
|
if (!Number.isFinite(at) || at > nowMs) continue;
|
|
if (r.category === "server" && r.targetType === "server" && (r.action === "went_offline" || r.action === "back_online")) {
|
|
const id = Number(r.targetId);
|
|
const list = serverEvents.get(id) ?? [];
|
|
list.push({ at, down: r.action === "went_offline" });
|
|
serverEvents.set(id, list);
|
|
} else if (r.category === "integration" && r.targetType === "integration_type" && r.targetId && (r.action === "went_failing" || r.action === "recovered")) {
|
|
const list = sourceEvents.get(r.targetId) ?? [];
|
|
list.push({ at, down: r.action === "went_failing" });
|
|
sourceEvents.set(r.targetId, list);
|
|
}
|
|
}
|
|
return { serverEvents, sourceEvents };
|
|
}
|
|
|
|
export async function getUptime(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<UptimeReport> {
|
|
const nowMs = now.getTime();
|
|
const { serverEvents, sourceEvents } = await loadTransitions(nowMs);
|
|
|
|
const out: UptimeReport = { days, endsAt: now.toISOString(), servers: {}, integrations: {} };
|
|
|
|
for (const s of await db.select({ id: servers.id, createdAt: servers.createdAt, lastSeenAt: servers.lastSeenAt }).from(servers)) {
|
|
const events = serverEvents.get(s.id) ?? [];
|
|
// A server that has never reported and never been seen to stop has no history to speak of.
|
|
if (!s.lastSeenAt && events.length === 0) continue;
|
|
const created = Date.parse(auditTimeToIso(s.createdAt));
|
|
const result = series(events, Number.isFinite(created) ? created : nowMs, nowMs, days);
|
|
if (result) out.servers[s.id] = result;
|
|
}
|
|
|
|
const since = await ensureHistoryStart(nowMs);
|
|
for (const i of await db.select({ id: integrations.id, type: integrations.type, createdAt: integrations.createdAt }).from(integrations)) {
|
|
const created = Date.parse(auditTimeToIso(i.createdAt));
|
|
const knownFrom = Math.max(Number.isFinite(created) ? created : 0, since);
|
|
const result = series(sourceEvents.get(i.type) ?? [], knownFrom, nowMs, days);
|
|
if (result) out.integrations[i.id] = result;
|
|
}
|
|
return out;
|
|
}
|