Files
Homelab-manager/server/src/services/uptimeHistory.ts
T
bobbanandClaude Sonnet 5.5 3b31b16796 Service-level targets with error budgets, and quarterly and annual reports
Monitoring -> Service levels: a target (99.9 %) on an endpoint check, a server or an
integration, measured over calendar months. Shows the figure, downtime used, the
error budget left, and how the last six months went. 80 % of the budget used is a
warning and an overspent budget a critical alert. Endpoints are measured day by day,
servers and integrations to the minute from the audit log.

The report page gains Quarter and Year (same figures, with a Service levels section),
and two settings send the quarterly and annual report after the month that ends them.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-21 22:20:07 +02:00

234 lines
9.9 KiB
TypeScript

/**
* How each server and integration has done over the last weeks, as one figure per day: how long it was up, how long down, and for
* how much of the day anything is known. It is worked out from the moments the health checks wrote down in the audit log (a
* server stopped reporting / reported again; an integration's calls started failing / worked again), so it needs no table of its
* own — and it can only be as old as those records.
*/
import { inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { auditLog, integrations, servers } from "../db/schema.js";
import { auditTimeToIso } from "./serverActivity.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
const DAY_MS = 24 * 3600_000;
const HISTORY_FLAG = "integration_history_since";
export const DEFAULT_DAYS = 30;
export interface Transition {
/** Epoch ms. */
at: number;
down: boolean;
}
/** Minutes of one 24-hour stretch that are known, and of those, minutes spent down. */
export interface UptimeBucket {
k: number;
d: number;
}
export interface UptimeSeries {
/** Oldest first; the last bucket ends now. */
buckets: UptimeBucket[];
/** Share of the known time spent up over the whole period. */
percent: number;
}
export interface Segment {
from: number;
to: number;
down: boolean;
}
/**
* Splits [knownFrom, now] into up and down stretches from the recorded transitions. Before the first transition the thing is taken to have been in the opposite state
* (a "went offline" means it was up until then); with no transitions at all it has been up the whole time.
*/
export function segmentsOf(transitions: Transition[], knownFrom: number, now: number): Segment[] {
const sorted = [...transitions].sort((a, b) => a.at - b.at);
// State at knownFrom: after any transitions that happened before it, otherwise the opposite of the first one that follows.
let state = false;
let i = 0;
while (i < sorted.length && sorted[i].at <= knownFrom) state = sorted[i++].down;
if (i === 0 && sorted.length > 0) state = !sorted[0].down;
const segments: Segment[] = [];
let cursor = Math.min(knownFrom, now);
for (; i < sorted.length; i++) {
const t = Math.min(sorted[i].at, now);
if (t > cursor) segments.push({ from: cursor, to: t, down: state });
cursor = Math.max(cursor, t);
state = sorted[i].down;
}
if (now > cursor) segments.push({ from: cursor, to: now, down: state });
return segments;
}
/** Minutes known, and of those spent down, within [from, to]. */
export function minutesWithin(segments: Segment[], from: number, to: number): { known: number; down: number } {
let known = 0;
let down = 0;
for (const seg of segments) {
const overlap = Math.min(seg.to, to) - Math.max(seg.from, from);
if (overlap <= 0) continue;
known += overlap;
if (seg.down) down += overlap;
}
return { known: Math.round(known / 60_000), down: Math.round(down / 60_000) };
}
/** The segments laid over `days` consecutive 24-hour buckets ending at `now`. */
export function computeBuckets(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeBucket[] {
const start = now - days * DAY_MS;
const segments = segmentsOf(transitions, knownFrom, now);
const known = new Array<number>(days).fill(0);
const down = new Array<number>(days).fill(0);
for (const seg of segments) {
if (seg.to <= start) continue;
const first = Math.max(0, Math.floor((Math.max(seg.from, start) - start) / DAY_MS));
for (let b = first; b < days; b++) {
const bStart = start + b * DAY_MS;
if (bStart >= seg.to) break;
const overlap = Math.min(seg.to, bStart + DAY_MS) - Math.max(seg.from, bStart);
if (overlap <= 0) continue;
known[b] += overlap;
if (seg.down) down[b] += overlap;
}
}
return known.map((k, b) => ({ k: Math.round(k / 60_000), d: Math.round(down[b] / 60_000) }));
}
/** Share of the known time spent up, to one decimal; null when nothing is known. */
export function uptimePercent(buckets: UptimeBucket[]): number | null {
const known = buckets.reduce((n, b) => n + b.k, 0);
if (known === 0) return null;
const down = buckets.reduce((n, b) => n + b.d, 0);
return Math.round((1 - down / known) * 1000) / 10;
}
function series(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeSeries | null {
const buckets = computeBuckets(transitions, knownFrom, now, days);
const percent = uptimePercent(buckets);
return percent === null ? null : { buckets, percent };
}
/**
* Integrations' failing / working-again moments only exist from the version that began writing them, so nothing earlier can be claimed.
* The first time anything asks, that moment is written down and kept.
*/
let historyStart: number | null = null;
export async function ensureHistoryStart(now: number = Date.now()): Promise<number> {
if (historyStart !== null) return historyStart;
const stored = await getInternalFlag(HISTORY_FLAG);
const parsed = stored ? Date.parse(stored) : NaN;
if (Number.isFinite(parsed)) {
historyStart = parsed;
return parsed;
}
await setInternalFlag(HISTORY_FLAG, new Date(now).toISOString());
historyStart = now;
return now;
}
/** For tests: forget the remembered start. */
export function resetHistoryStartForTests(): void {
historyStart = null;
}
export interface UptimeReport {
days: number;
/** ISO — the end of the last bucket. */
endsAt: string;
servers: Record<number, UptimeSeries>;
integrations: Record<number, UptimeSeries>;
}
/** How long a server or integration was known, and down, between two moments. */
export interface UptimeTotals {
known: number;
down: number;
}
export interface UptimeRange {
servers: Record<number, UptimeTotals>;
integrations: Record<number, UptimeTotals>;
}
/**
* Exactly how long each server and integration was down between `from` and `to` (the end is clipped to now), to the minute — for a target that must be measured over a
* calendar period rather than in whole days back from now. Only the ones known for some of it are listed.
*/
export async function getUptimeRange(from: Date, to: Date, now: Date = new Date()): Promise<UptimeRange> {
const nowMs = now.getTime();
const end = Math.min(to.getTime(), nowMs);
const start = from.getTime();
const out: UptimeRange = { servers: {}, integrations: {} };
if (end <= start) return out;
const { serverEvents, sourceEvents } = await loadTransitions(nowMs);
for (const s of await db.select({ id: servers.id, createdAt: servers.createdAt, lastSeenAt: servers.lastSeenAt }).from(servers)) {
const events = serverEvents.get(s.id) ?? [];
if (!s.lastSeenAt && events.length === 0) continue;
const created = Date.parse(auditTimeToIso(s.createdAt));
const totals = minutesWithin(segmentsOf(events, Number.isFinite(created) ? created : nowMs, nowMs), start, end);
if (totals.known > 0) out.servers[s.id] = totals;
}
const since = await ensureHistoryStart(nowMs);
for (const i of await db.select({ id: integrations.id, type: integrations.type, createdAt: integrations.createdAt }).from(integrations)) {
const created = Date.parse(auditTimeToIso(i.createdAt));
const knownFrom = Math.max(Number.isFinite(created) ? created : 0, since);
const totals = minutesWithin(segmentsOf(sourceEvents.get(i.type) ?? [], knownFrom, nowMs), start, end);
if (totals.known > 0) out.integrations[i.id] = totals;
}
return out;
}
/** The recorded moments servers went away and came back, and integrations started failing and worked again. */
async function loadTransitions(nowMs: number): Promise<{ serverEvents: Map<number, Transition[]>; sourceEvents: Map<string, Transition[]> }> {
const rows = await db.select().from(auditLog).where(inArray(auditLog.action, ["went_offline", "back_online", "went_failing", "recovered"]));
const serverEvents = new Map<number, Transition[]>();
const sourceEvents = new Map<string, Transition[]>();
for (const r of rows) {
const at = Date.parse(auditTimeToIso(r.createdAt));
if (!Number.isFinite(at) || at > nowMs) continue;
if (r.category === "server" && r.targetType === "server" && (r.action === "went_offline" || r.action === "back_online")) {
const id = Number(r.targetId);
const list = serverEvents.get(id) ?? [];
list.push({ at, down: r.action === "went_offline" });
serverEvents.set(id, list);
} else if (r.category === "integration" && r.targetType === "integration_type" && r.targetId && (r.action === "went_failing" || r.action === "recovered")) {
const list = sourceEvents.get(r.targetId) ?? [];
list.push({ at, down: r.action === "went_failing" });
sourceEvents.set(r.targetId, list);
}
}
return { serverEvents, sourceEvents };
}
export async function getUptime(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<UptimeReport> {
const nowMs = now.getTime();
const { serverEvents, sourceEvents } = await loadTransitions(nowMs);
const out: UptimeReport = { days, endsAt: now.toISOString(), servers: {}, integrations: {} };
for (const s of await db.select({ id: servers.id, createdAt: servers.createdAt, lastSeenAt: servers.lastSeenAt }).from(servers)) {
const events = serverEvents.get(s.id) ?? [];
// A server that has never reported and never been seen to stop has no history to speak of.
if (!s.lastSeenAt && events.length === 0) continue;
const created = Date.parse(auditTimeToIso(s.createdAt));
const result = series(events, Number.isFinite(created) ? created : nowMs, nowMs, days);
if (result) out.servers[s.id] = result;
}
const since = await ensureHistoryStart(nowMs);
for (const i of await db.select({ id: integrations.id, type: integrations.type, createdAt: integrations.createdAt }).from(integrations)) {
const created = Date.parse(auditTimeToIso(i.createdAt));
const knownFrom = Math.max(Number.isFinite(created) ? created : 0, since);
const result = series(sourceEvents.get(i.type) ?? [], knownFrom, nowMs, days);
if (result) out.integrations[i.id] = result;
}
return out;
}