Service-level targets with error budgets, and quarterly and annual reports
Monitoring -> Service levels: a target (99.9 %) on an endpoint check, a server or an integration, measured over calendar months. Shows the figure, downtime used, the error budget left, and how the last six months went. 80 % of the budget used is a warning and an overspent budget a critical alert. Endpoints are measured day by day, servers and integrations to the minute from the audit log. The report page gains Quarter and Year (same figures, with a Service levels section), and two settings send the quarterly and annual report after the month that ends them. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
31044b2b64
commit
3b31b16796
31 files changed
+7706
-61
No files matched your search
@@ -73,6 +73,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
npm: { label: "Nginx Proxy Manager", link: "/proxy-hosts" },
|
||||
homeassistant: { label: "Home Assistant", link: "/home-assistant" },
|
||||
ups: { label: "UPS", link: "/ups" },
|
||||
slo: { label: "Service level", link: "/service-levels" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -105,6 +106,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: m[1], id: Number(m[2]), category: "automation" };
|
||||
m = /^ups-(?:battery|low|replace|load):(\d+)$/.exec(key);
|
||||
if (m) return { type: "ups", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^slo:(\d+)$/.exec(key);
|
||||
if (m) return { type: "slo", id: Number(m[1]), category: "monitoring" };
|
||||
return { type: "server", id: null, category: "disk" };
|
||||
}
|
||||
|
||||
@@ -227,12 +230,13 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
|
||||
if (held.has("server:rules")) cannotCheck("Alert rules", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups):(\d+)(?::(.+))?$/.exec(h);
|
||||
|
||||
@@ -41,6 +41,7 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
|
||||
metrics_token: "settings",
|
||||
api_token: "settings",
|
||||
alert_rule: "alerts",
|
||||
service_level: "endpoints",
|
||||
todo: "other",
|
||||
note: "servers",
|
||||
webhook: "settings",
|
||||
@@ -181,6 +182,12 @@ export function describeChange(category: string, action: string, detail: Record<
|
||||
return { title: detail.change === "undo" ? "A change to several pieces of hardware was undone" : `${typeof detail.changed === "number" ? detail.changed : "Several"} piece${detail.changed === 1 ? "" : "s"} of hardware changed at once`, tone: "neutral", detail: genericSummary(detail, ["ids", "changed", "change"]) };
|
||||
case "webhook/retry":
|
||||
return { title: `A webhook message was sent again${name ? `: ${name}` : str(detail.label) ? `: ${str(detail.label)}` : ""}`, tone: detail.ok === false ? "bad" : "neutral", detail: str(detail.event) };
|
||||
case "service_level/create":
|
||||
return { title: `Service-level target set${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.target === "number" ? `${detail.target}%` : null };
|
||||
case "service_level/update":
|
||||
return { title: `Service-level target changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.target === "number" ? `${typeof detail.targetFrom === "number" ? `${detail.targetFrom}% → ` : ""}${detail.target}%` : null };
|
||||
case "service_level/delete":
|
||||
return { title: `Service-level target removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "alert_rule/create":
|
||||
return { title: `Alert rule added${name ? `: ${name}` : ""}`, tone: "neutral", detail: genericSummary(detail) };
|
||||
case "alert_rule/update":
|
||||
|
||||
@@ -17,6 +17,7 @@ import { parseJson, type AgentPort } from "./agentPorts.js";
|
||||
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
|
||||
import { computeRuleFindings, findingKey, listRules, type RuleFinding } from "./alertRules.js";
|
||||
import { baselineKey, computeBaselineFindings, type BaselineFinding } from "./baselines.js";
|
||||
import { levelFindings, levelKey, type LevelFinding } from "./serviceLevels.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
|
||||
|
||||
@@ -81,6 +82,8 @@ export interface HealthSnapshot {
|
||||
rules?: RuleFinding[];
|
||||
/** What is clearly higher than usual for this hour of the week, from each server's own history (services/baselines.ts). */
|
||||
baselines?: BaselineFinding[];
|
||||
/** Service-level targets whose error budget for the month is mostly or wholly used (services/serviceLevels.ts). */
|
||||
levels?: LevelFinding[];
|
||||
}
|
||||
export interface Thresholds {
|
||||
serverOfflineMinutes: number;
|
||||
@@ -183,6 +186,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
}
|
||||
}
|
||||
|
||||
// A service-level target that has used most of its month's error budget, or all of it.
|
||||
for (const f of snapshot.levels ?? []) out.push({ key: levelKey(f), source: "slo", message: f.message, severity: f.severity });
|
||||
|
||||
for (const u of snapshot.unifi ?? []) {
|
||||
const limitMs = (thresholds.deviceOfflineMinutes ?? 10) * 60_000;
|
||||
for (const d of u.devices) {
|
||||
@@ -402,6 +408,14 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
}
|
||||
}
|
||||
|
||||
// Service-level targets: how much of the month's error budget is used. A failure here costs only this check.
|
||||
try {
|
||||
snapshot.levels = await levelFindings();
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge the service-level targets:", err instanceof Error ? err.message : err);
|
||||
held.add("slo");
|
||||
}
|
||||
|
||||
const rows = await db
|
||||
.select({ id: integrations.id, name: integrations.name, type: integrations.type })
|
||||
.from(integrations)
|
||||
|
||||
@@ -7,6 +7,7 @@ import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { heartbeatState, parseStamp } from "./heartbeats.js";
|
||||
import { env } from "../env.js";
|
||||
@@ -58,6 +59,17 @@ class Exposition {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each service-level target, for /metrics: its target, how it is doing this month and how much error budget is left. */
|
||||
async function addServiceLevelMetrics(out: Exposition, now: Date) {
|
||||
for (const s of await levelStatuses(now, { history: false })) {
|
||||
const labels = { name: s.name, subject: s.subjectName ?? "", type: s.subjectType };
|
||||
out.add("homelab_service_level_target_percent", "gauge", "The availability a service-level target asks for.", s.target, labels);
|
||||
if (s.measure.percent === null) continue;
|
||||
out.add("homelab_service_level_availability_percent", "gauge", "How available the thing has been so far this calendar month.", s.measure.percent, labels);
|
||||
out.add("homelab_service_level_error_budget_remaining_minutes", "gauge", "Downtime still allowed this month before the target is missed (negative once it is).", Math.round(s.budget.remainingMinutes * 10) / 10, labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Each UPS, for /metrics: whether it is on battery, how full the battery is, how long it would last and how loaded it is. */
|
||||
async function addUpsMetrics(out: Exposition, now: Date) {
|
||||
const fresh = new Set<number>();
|
||||
@@ -239,6 +251,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
|
||||
await addEndpointMetrics(out, now);
|
||||
await addUpsMetrics(out, now);
|
||||
await addServiceLevelMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* The monthly report: how a month went — availability of servers, integrations and endpoints, incidents and how long they took, updates applied (by hand and by
|
||||
* The monthly report (and, by the same figures over longer stretches, the quarterly and annual ones): how a month went — availability of servers, integrations and endpoints, incidents and how long they took, updates applied (by hand and by
|
||||
* patch plans), restore tests and the state of the backups, and what was renewed. Everything in it the app already knows; this gathers it for one calendar month
|
||||
* (on the clock of the notification time zone) into one structure that the page shows, prints and exports, and into a plain-text version that can be sent like the
|
||||
* weekly digest.
|
||||
@@ -18,6 +18,7 @@ import { renewalsBetween } from "./renewals.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
import { spanWords } from "./weeklyDigest.js";
|
||||
import { levelResultsFor, type PeriodResult } from "./serviceLevels.js";
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
const MAX_LISTED = 12;
|
||||
@@ -38,10 +39,14 @@ export interface EndpointRow {
|
||||
checks: number;
|
||||
}
|
||||
|
||||
export type PeriodKind = "month" | "quarter" | "year";
|
||||
|
||||
export interface MonthlyReport {
|
||||
/** "2026-09". */
|
||||
/** What stretch it is about: a month (id "2026-09"), a quarter ("2026-Q3") or a year ("2026"). */
|
||||
period: PeriodKind;
|
||||
/** The id of that stretch. (Named for the month, which is what it was made for.) */
|
||||
month: string;
|
||||
/** "September 2026". */
|
||||
/** "September 2026", "Q3 2026", "2026". */
|
||||
label: string;
|
||||
/** ISO — the start of the month and its end (or now, for a month that is not over). */
|
||||
from: string;
|
||||
@@ -52,6 +57,8 @@ export interface MonthlyReport {
|
||||
servers: AvailabilityRow[];
|
||||
integrations: AvailabilityRow[];
|
||||
endpoints: EndpointRow[];
|
||||
/** How each service-level target did over the stretch (those with a figure for it). */
|
||||
serviceLevels: PeriodResult[];
|
||||
incidents: {
|
||||
began: number;
|
||||
resolved: number;
|
||||
@@ -94,6 +101,17 @@ function monthOf(now: Date, tz: string): { year: number; month: number } {
|
||||
}
|
||||
|
||||
export const monthId = (year: number, month: number) => `${year}-${pad(month)}`;
|
||||
|
||||
export function isQuarterId(v: unknown): boolean {
|
||||
return typeof v === "string" && /^\d{4}-Q[1-4]$/.test(v);
|
||||
}
|
||||
export function isYearId(v: unknown): boolean {
|
||||
return typeof v === "string" && /^\d{4}$/.test(v);
|
||||
}
|
||||
export const isPeriodId = (v: unknown): v is string => isMonthId(v) || isQuarterId(v) || isYearId(v);
|
||||
export function periodKind(id: string): PeriodKind {
|
||||
return isQuarterId(id) ? "quarter" : isYearId(id) ? "year" : "month";
|
||||
}
|
||||
const shift = (year: number, month: number, by: number) => {
|
||||
const index = year * 12 + (month - 1) + by;
|
||||
return { year: Math.floor(index / 12), month: (index % 12) + 1 };
|
||||
@@ -121,6 +139,61 @@ export function recentMonths(now: Date, tz: string, count = 12): { id: string; l
|
||||
});
|
||||
}
|
||||
|
||||
export function periodLabel(id: string): string {
|
||||
if (isQuarterId(id)) return `Q${id.slice(6)} ${id.slice(0, 4)}`;
|
||||
return isYearId(id) ? id : monthLabel(id);
|
||||
}
|
||||
|
||||
/** The year and quarter (1–4) `now` is in. */
|
||||
function quarterOf(now: Date, tz: string): { year: number; quarter: number } {
|
||||
const { year, month } = monthOf(now, tz);
|
||||
return { year, quarter: Math.ceil(month / 3) };
|
||||
}
|
||||
|
||||
/** The last whole quarter ("2026-Q2" in July), and the last whole year. */
|
||||
export function lastFullQuarter(now: Date, tz: string): string {
|
||||
const { year, quarter } = quarterOf(now, tz);
|
||||
return quarter === 1 ? `${year - 1}-Q4` : `${year}-Q${quarter - 1}`;
|
||||
}
|
||||
export const lastFullYear = (now: Date, tz: string) => String(monthOf(now, tz).year - 1);
|
||||
|
||||
/** What can be asked for: this month and eleven before it, this quarter and seven before it, this year and two before it — newest first. */
|
||||
export function recentPeriods(now: Date, tz: string): { months: { id: string; label: string }[]; quarters: { id: string; label: string }[]; years: { id: string; label: string }[] } {
|
||||
const { year, quarter } = quarterOf(now, tz);
|
||||
const quarters = Array.from({ length: 8 }, (_, i) => {
|
||||
const index = year * 4 + (quarter - 1) - i;
|
||||
const id = `${Math.floor(index / 4)}-Q${(index % 4) + 1}`;
|
||||
return { id, label: periodLabel(id) };
|
||||
});
|
||||
const years = Array.from({ length: 3 }, (_, i) => ({ id: String(year - i), label: String(year - i) }));
|
||||
return { months: recentMonths(now, tz), quarters, years };
|
||||
}
|
||||
|
||||
export interface PeriodBounds {
|
||||
start: Date;
|
||||
end: Date;
|
||||
/** The first UTC day of it and of what follows, as "YYYY-MM-DD", for the figures kept per UTC day. */
|
||||
firstDay: string;
|
||||
nextFirstDay: string;
|
||||
}
|
||||
|
||||
/** The instants a month, quarter or year begins and ends at, on the clock of `tz`. */
|
||||
export function periodBounds(id: string, tz: string): PeriodBounds {
|
||||
if (isYearId(id)) {
|
||||
const y = Number(id);
|
||||
return { start: zonedTime(y, 1, 1, 0, 0, tz), end: zonedTime(y + 1, 1, 1, 0, 0, tz), firstDay: `${y}-01-01`, nextFirstDay: `${y + 1}-01-01` };
|
||||
}
|
||||
if (isQuarterId(id)) {
|
||||
const y = Number(id.slice(0, 4));
|
||||
const first = (Number(id.slice(6)) - 1) * 3 + 1;
|
||||
const next = shift(y, first, 3);
|
||||
return { start: zonedTime(y, first, 1, 0, 0, tz), end: zonedTime(next.year, next.month, 1, 0, 0, tz), firstDay: `${y}-${pad(first)}-01`, nextFirstDay: `${monthId(next.year, next.month)}-01` };
|
||||
}
|
||||
const [y, m] = id.split("-").map(Number);
|
||||
const next = shift(y, m, 1);
|
||||
return { ...monthBounds(id, tz), firstDay: `${id}-01`, nextFirstDay: `${monthId(next.year, next.month)}-01` };
|
||||
}
|
||||
|
||||
/** The instants a month begins and ends at, on the clock of `tz`. */
|
||||
export function monthBounds(id: string, tz: string): { start: Date; end: Date } {
|
||||
const [y, m] = id.split("-").map(Number);
|
||||
@@ -135,7 +208,8 @@ const round1 = (n: number) => Math.round(n * 10) / 10;
|
||||
export async function buildMonthlyReport(id: string, now: Date = new Date()): Promise<MonthlyReport> {
|
||||
const settings = await getSettings();
|
||||
const tz = settings.notifications.timezone || "UTC";
|
||||
const { start, end: fullEnd } = monthBounds(id, tz);
|
||||
const bounds = periodBounds(id, tz);
|
||||
const { start, end: fullEnd } = bounds;
|
||||
const partial = fullEnd.getTime() > now.getTime();
|
||||
const end = partial ? now : fullEnd;
|
||||
const days = Math.max(1, Math.ceil((end.getTime() - start.getTime()) / DAY_MS));
|
||||
@@ -154,10 +228,8 @@ export async function buildMonthlyReport(id: string, now: Date = new Date()): Pr
|
||||
|
||||
// Endpoints: from the daily figures, for the days the month covers. The figures are per UTC day, so a day at either end can belong to a neighbouring month
|
||||
// by a few hours (more than the time zone is ahead of UTC); a month that is not over takes in today.
|
||||
const [yy, mm] = id.split("-").map(Number);
|
||||
const following = shift(yy, mm, 1);
|
||||
const firstDay = `${id}-01`;
|
||||
const lastDayExclusive = partial ? new Date(now.getTime() + DAY_MS).toISOString().slice(0, 10) : `${monthId(following.year, following.month)}-01`;
|
||||
const firstDay = bounds.firstDay;
|
||||
const lastDayExclusive = partial ? new Date(now.getTime() + DAY_MS).toISOString().slice(0, 10) : bounds.nextFirstDay;
|
||||
const dailyRows = await db.select().from(endpointDaily).where(and(gte(endpointDaily.day, firstDay), lt(endpointDaily.day, lastDayExclusive)));
|
||||
const checkNames = new Map((await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks)).map((c) => [c.id, c.name]));
|
||||
const perCheck = new Map<number, { total: number; ok: number; msSum: number; msCount: number }>();
|
||||
@@ -175,7 +247,7 @@ export async function buildMonthlyReport(id: string, now: Date = new Date()): Pr
|
||||
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
|
||||
|
||||
// Incidents.
|
||||
const all = await listIncidents({ now: end, limit: 3000 });
|
||||
const all = await listIncidents({ now: end, limit: 20000 });
|
||||
const inMonth = (iso: string) => iso >= startIso && iso < endIso;
|
||||
const began = all.filter((i) => inMonth(i.openedAt));
|
||||
const resolved = all.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
|
||||
@@ -209,8 +281,9 @@ export async function buildMonthlyReport(id: string, now: Date = new Date()): Pr
|
||||
const renewalRows = await renewalsBetween(start, end);
|
||||
|
||||
return {
|
||||
period: periodKind(id),
|
||||
month: id,
|
||||
label: monthLabel(id),
|
||||
label: periodLabel(id),
|
||||
from: startIso,
|
||||
to: endIso,
|
||||
partial,
|
||||
@@ -218,6 +291,7 @@ export async function buildMonthlyReport(id: string, now: Date = new Date()): Pr
|
||||
servers: rowsOf(uptime.servers, serverNames),
|
||||
integrations: rowsOf(uptime.integrations, integrationNames),
|
||||
endpoints: endpointRows,
|
||||
serviceLevels: await levelResultsFor(start, fullEnd, now),
|
||||
incidents: {
|
||||
began: began.length,
|
||||
resolved: resolved.length,
|
||||
@@ -274,6 +348,13 @@ export function monthlyReportText(r: MonthlyReport): { title: string; message: s
|
||||
if (integrationAvg !== null) avail.push(`Integrations: ${integrationAvg}% on average.`);
|
||||
lines("Availability", avail);
|
||||
|
||||
if (r.serviceLevels.length > 0) {
|
||||
const missed = r.serviceLevels.filter((l) => !l.met);
|
||||
const body = [`${r.serviceLevels.length - missed.length} of ${plural(r.serviceLevels.length, "target")} met${r.partial ? " so far" : ""}.`];
|
||||
for (const l of r.serviceLevels.slice(0, 8)) body.push(`- ${l.name}: ${l.percent}% against ${l.target}% — ${l.met ? "met" : "MISSED"} (down ${l.downMinutes > 0 ? spanWords(l.downMinutes) : "not at all"}, ${l.allowedMinutes >= 1 ? spanWords(l.allowedMinutes) : "under a minute"} allowed)`);
|
||||
lines("Service levels", body);
|
||||
}
|
||||
|
||||
const inc = r.incidents;
|
||||
if (inc.began > 0 || inc.resolved > 0 || inc.stillOpen > 0) {
|
||||
const body = [`${plural(inc.began, "incident")} began, ${inc.resolved} ended, ${inc.stillOpen} still open.`];
|
||||
@@ -298,6 +379,7 @@ export function monthlyReportText(r: MonthlyReport): { title: string; message: s
|
||||
|
||||
lines("Renewed", r.renewals.slice(0, 10).map((x) => `- ${x.kind === "domain" ? "Domain" : "Certificate"} ${x.subject}: ${x.fromDate} → ${x.toDate}`));
|
||||
|
||||
const title = `Homelab monthly report — ${r.label}${r.partial ? " (so far)" : ""}`;
|
||||
return { title, message: sections.length > 0 ? sections.join("\n\n") : "Nothing was recorded for this month." };
|
||||
const noun = r.period === "year" ? "annual" : r.period === "quarter" ? "quarterly" : "monthly";
|
||||
const title = `Homelab ${noun} report — ${r.label}${r.partial ? " (so far)" : ""}`;
|
||||
return { title, message: sections.length > 0 ? sections.join("\n\n") : `Nothing was recorded for this ${r.period}.` };
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import schedule from "node-schedule";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { notify } from "./notify.js";
|
||||
import { buildMonthlyReport, lastFullMonth, monthlyReportText } from "./monthlyReport.js";
|
||||
import { buildMonthlyReport, lastFullMonth, lastFullQuarter, lastFullYear, monthlyReportText } from "./monthlyReport.js";
|
||||
|
||||
let currentJob: schedule.Job | null = null;
|
||||
|
||||
@@ -12,11 +12,25 @@ export function cronForMonthlyReport(day: number, time: string): string {
|
||||
return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} ${d} * *`;
|
||||
}
|
||||
|
||||
/** Builds last month's report and sends it through the channels (held with other notifications during quiet hours). */
|
||||
/**
|
||||
* Builds last month's report and sends it through the channels (held with other notifications during quiet hours). When that month closed a quarter and the quarterly
|
||||
* report is on, the quarter's report follows it; when it closed the year and the annual one is on, the year's.
|
||||
*/
|
||||
export async function sendMonthlyReport(now: Date = new Date()): Promise<{ title: string; message: string }> {
|
||||
const tz = (await getSettings()).notifications.timezone || "UTC";
|
||||
const text = monthlyReportText(await buildMonthlyReport(lastFullMonth(now, tz), now));
|
||||
const settings = await getSettings();
|
||||
const tz = settings.notifications.timezone || "UTC";
|
||||
const month = lastFullMonth(now, tz);
|
||||
const text = monthlyReportText(await buildMonthlyReport(month, now));
|
||||
await notify(text.title, text.message);
|
||||
const closing = Number(month.slice(5));
|
||||
if (settings.monthlyReport.quarterly && closing % 3 === 0) {
|
||||
const q = monthlyReportText(await buildMonthlyReport(lastFullQuarter(now, tz), now));
|
||||
await notify(q.title, q.message);
|
||||
}
|
||||
if (settings.monthlyReport.annual && closing === 12) {
|
||||
const y = monthlyReportText(await buildMonthlyReport(lastFullYear(now, tz), now));
|
||||
await notify(y.title, y.message);
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
/**
|
||||
* Service-level targets: "99.9 % for Gitea", and what is left of the error budget this month. A target is set on something the app already keeps the record of — an
|
||||
* endpoint check, a server, an integration — and is measured over calendar months (on the clock of the notification time zone):
|
||||
*
|
||||
* allowed downtime = minutes in the month × (1 − target) — that is the error budget
|
||||
* used = minutes the thing was down so far
|
||||
* left = allowed − used
|
||||
*
|
||||
* An endpoint's downtime is worked out from its daily check figures (each day's share of failed checks, over the part of the day that is within the period); a server's and an integration's
|
||||
* from the moments the audit log recorded them going away and coming back, to the minute. Planned maintenance is *not* taken out: down is down. Nothing is guessed — a
|
||||
* thing with no record for the period has no figure, and says so.
|
||||
*
|
||||
* A target whose budget is mostly used (80 %) is a warning, and one that is overspent a critical alert, until the month turns over and the budget starts again.
|
||||
*/
|
||||
import { and, gte, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, endpointDaily, integrations, servers, serviceLevels } from "../db/schema.js";
|
||||
import { monthBounds, recentMonths } from "./monthlyReport.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptimeRange, type UptimeRange } from "./uptimeHistory.js";
|
||||
import { spanWords } from "./weeklyDigest.js";
|
||||
|
||||
export const SUBJECT_TYPES = ["endpoint", "server", "integration"] as const;
|
||||
export type SubjectType = (typeof SUBJECT_TYPES)[number];
|
||||
export const MAX_LEVELS = 50;
|
||||
export const MIN_TARGET = 90;
|
||||
export const MAX_TARGET = 99.999;
|
||||
/** The share of the budget used at which it is a warning. */
|
||||
export const WARN_AT_PERCENT = 80;
|
||||
export const HISTORY_MONTHS = 6;
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
export type LevelRow = typeof serviceLevels.$inferSelect;
|
||||
export type BudgetState = "ok" | "at_risk" | "breached" | "no_data";
|
||||
|
||||
export interface Measure {
|
||||
/** Share of the known time spent up, in percent (three decimals); null when nothing is known. */
|
||||
percent: number | null;
|
||||
downMinutes: number;
|
||||
knownMinutes: number;
|
||||
}
|
||||
|
||||
export interface Budget {
|
||||
allowedMinutes: number;
|
||||
/** Of the budget, in percent (can pass 100). Null without a figure. */
|
||||
usedPercent: number | null;
|
||||
/** Negative when overspent. */
|
||||
remainingMinutes: number;
|
||||
state: BudgetState;
|
||||
/** Of the budget, at the pace so far, by the end of the period; null before a day has passed or without a figure. */
|
||||
projectedPercent: number | null;
|
||||
}
|
||||
|
||||
const round3 = (n: number) => Math.round(n * 1000) / 1000;
|
||||
|
||||
/** Pure: the error budget of a target over a period of `totalMinutes`, and where it stands after `elapsedMinutes` with `downMinutes` used. */
|
||||
export function judgeBudget(target: number, totalMinutes: number, elapsedMinutes: number, measure: Measure): Budget {
|
||||
const allowed = totalMinutes * (1 - target / 100);
|
||||
if (measure.percent === null) return { allowedMinutes: allowed, usedPercent: null, remainingMinutes: allowed, state: "no_data", projectedPercent: null };
|
||||
const used = measure.downMinutes;
|
||||
const usedPercent = allowed > 0 ? (used / allowed) * 100 : used > 0 ? Infinity : 0;
|
||||
const state: BudgetState = used > allowed ? "breached" : usedPercent >= WARN_AT_PERCENT ? "at_risk" : "ok";
|
||||
const projected = elapsedMinutes >= 1440 && allowed > 0 ? ((used / elapsedMinutes) * totalMinutes / allowed) * 100 : null;
|
||||
return { allowedMinutes: allowed, usedPercent: Number.isFinite(usedPercent) ? Math.round(usedPercent * 10) / 10 : 999, remainingMinutes: allowed - used, state, projectedPercent: projected === null ? null : Math.round(projected) };
|
||||
}
|
||||
|
||||
/** A budget in words: "43 min", "4 h". Below a minute it is "under a minute". */
|
||||
export const budgetWords = (minutes: number) => (minutes < 1 ? "under a minute" : spanWords(minutes));
|
||||
|
||||
/** Pure: what an alert about a budget says. */
|
||||
export function budgetMessage(name: string, target: number, periodLabel: string, b: Budget, m: Measure): string {
|
||||
if (b.state === "breached") return `${name} has missed its ${target}% target for ${periodLabel}: down ${budgetWords(m.downMinutes)}, ${budgetWords(b.allowedMinutes)} allowed`;
|
||||
return `${name}: ${Math.round(b.usedPercent ?? 0)}% of the error budget for its ${target}% target is used up for ${periodLabel} — down ${budgetWords(m.downMinutes)} of ${budgetWords(b.allowedMinutes)} allowed, ${budgetWords(b.remainingMinutes)} left`;
|
||||
}
|
||||
|
||||
// ─── Measuring ──────────────────────────────────────────────────────────────
|
||||
|
||||
/** What is shared between the measurements of one request, so each stretch of the audit log and of the daily figures is read once. */
|
||||
export interface MeasureContext {
|
||||
now: Date;
|
||||
uptime: Map<string, Promise<UptimeRange>>;
|
||||
daily: Map<string, Promise<{ checkId: number; day: string; total: number; ok: number }[]>>;
|
||||
}
|
||||
|
||||
export const newContext = (now: Date = new Date()): MeasureContext => ({ now, uptime: new Map(), daily: new Map() });
|
||||
|
||||
const dayOf = (ms: number) => new Date(ms).toISOString().slice(0, 10);
|
||||
|
||||
/** How a thing did between two moments (the end is clipped to now). */
|
||||
export async function measure(type: SubjectType, subjectId: number, start: Date, end: Date, ctx: MeasureContext): Promise<Measure> {
|
||||
const to = Math.min(end.getTime(), ctx.now.getTime());
|
||||
const from = start.getTime();
|
||||
if (to <= from) return { percent: null, downMinutes: 0, knownMinutes: 0 };
|
||||
if (type === "endpoint") {
|
||||
const key = `${dayOf(from)}|${dayOf(to + DAY_MS - 1)}`;
|
||||
let rows = ctx.daily.get(key);
|
||||
if (!rows) {
|
||||
rows = (async () => db.select({ checkId: endpointDaily.checkId, day: endpointDaily.day, total: endpointDaily.total, ok: endpointDaily.ok }).from(endpointDaily).where(and(gte(endpointDaily.day, dayOf(from)), lt(endpointDaily.day, dayOf(to + DAY_MS - 1)))))();
|
||||
ctx.daily.set(key, rows);
|
||||
}
|
||||
// Day by day: the share of that day's checks that failed, over the part of the day that is within the period. Days with no checks are not known, so they count neither way.
|
||||
let known = 0;
|
||||
let down = 0;
|
||||
for (const r of await rows) {
|
||||
if (r.checkId !== subjectId || r.total <= 0) continue;
|
||||
const dayStart = Date.parse(`${r.day}T00:00:00Z`);
|
||||
const overlap = Math.min(to, dayStart + DAY_MS) - Math.max(from, dayStart);
|
||||
if (overlap <= 0) continue;
|
||||
known += overlap / 60_000;
|
||||
down += (1 - r.ok / r.total) * (overlap / 60_000);
|
||||
}
|
||||
if (known <= 0) return { percent: null, downMinutes: 0, knownMinutes: 0 };
|
||||
return { percent: round3((1 - down / known) * 100), downMinutes: Math.round(down), knownMinutes: Math.round(known) };
|
||||
}
|
||||
const key = `${from}|${to}`;
|
||||
let range = ctx.uptime.get(key);
|
||||
if (!range) {
|
||||
range = getUptimeRange(start, new Date(to), ctx.now);
|
||||
ctx.uptime.set(key, range);
|
||||
}
|
||||
const t = (await range)[type === "server" ? "servers" : "integrations"][subjectId];
|
||||
if (!t || t.known <= 0) return { percent: null, downMinutes: 0, knownMinutes: 0 };
|
||||
return { percent: round3((1 - t.down / t.known) * 100), downMinutes: t.down, knownMinutes: t.known };
|
||||
}
|
||||
|
||||
export interface LevelStatus {
|
||||
id: number;
|
||||
name: string;
|
||||
subjectType: SubjectType;
|
||||
subjectId: number;
|
||||
/** The name of the thing measured; null when it has been removed. */
|
||||
subjectName: string | null;
|
||||
target: number;
|
||||
alert: boolean;
|
||||
createdBy: string | null;
|
||||
period: { id: string; label: string; from: string; to: string; totalMinutes: number; elapsedMinutes: number };
|
||||
measure: Measure;
|
||||
budget: Budget;
|
||||
/** The months before this one, newest first. */
|
||||
history: { id: string; label: string; percent: number | null; downMinutes: number; allowedMinutes: number; met: boolean | null }[];
|
||||
}
|
||||
|
||||
async function subjectNames(): Promise<Record<SubjectType, Map<number, string>>> {
|
||||
const [e, s, i] = await Promise.all([db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks), db.select({ id: servers.id, name: servers.name }).from(servers), db.select({ id: integrations.id, name: integrations.name }).from(integrations)]);
|
||||
return { endpoint: new Map(e.map((r) => [r.id, r.name])), server: new Map(s.map((r) => [r.id, r.name])), integration: new Map(i.map((r) => [r.id, r.name])) };
|
||||
}
|
||||
|
||||
export async function listLevels(): Promise<LevelRow[]> {
|
||||
return (await db.select().from(serviceLevels)).sort((a, b) => a.name.localeCompare(b.name) || a.id - b.id);
|
||||
}
|
||||
|
||||
/** Every target, where this month's budget stands, and how each of the last months went. */
|
||||
export async function levelStatuses(now: Date = new Date(), options: { history?: boolean } = {}): Promise<LevelStatus[]> {
|
||||
const rows = await listLevels();
|
||||
if (rows.length === 0) return [];
|
||||
const tz = (await getSettings()).notifications.timezone || "UTC";
|
||||
const months = recentMonths(now, tz, HISTORY_MONTHS + 1);
|
||||
const current = months[0].id;
|
||||
const { start, end } = monthBounds(current, tz);
|
||||
const totalMinutes = (end.getTime() - start.getTime()) / 60_000;
|
||||
const elapsedMinutes = Math.max(0, (Math.min(now.getTime(), end.getTime()) - start.getTime()) / 60_000);
|
||||
const names = await subjectNames();
|
||||
const ctx = newContext(now);
|
||||
const out: LevelStatus[] = [];
|
||||
for (const r of rows) {
|
||||
const type = r.subjectType as SubjectType;
|
||||
const m = await measure(type, r.subjectId, start, end, ctx);
|
||||
const history: LevelStatus["history"] = [];
|
||||
if (options.history !== false) {
|
||||
for (const past of months.slice(1)) {
|
||||
const b = monthBounds(past.id, tz);
|
||||
const total = (b.end.getTime() - b.start.getTime()) / 60_000;
|
||||
const pm = await measure(type, r.subjectId, b.start, b.end, ctx);
|
||||
const allowed = total * (1 - r.target / 100);
|
||||
history.push({ id: past.id, label: past.label, percent: pm.percent, downMinutes: pm.downMinutes, allowedMinutes: allowed, met: pm.percent === null ? null : pm.downMinutes <= allowed });
|
||||
}
|
||||
}
|
||||
out.push({
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
subjectType: type,
|
||||
subjectId: r.subjectId,
|
||||
subjectName: names[type].get(r.subjectId) ?? null,
|
||||
target: r.target,
|
||||
alert: r.alert,
|
||||
createdBy: r.createdBy,
|
||||
period: { id: current, label: months[0].label, from: start.toISOString(), to: end.toISOString(), totalMinutes: Math.round(totalMinutes), elapsedMinutes: Math.round(elapsedMinutes) },
|
||||
measure: m,
|
||||
budget: judgeBudget(r.target, totalMinutes, elapsedMinutes, m),
|
||||
history,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface LevelFinding {
|
||||
levelId: number;
|
||||
name: string;
|
||||
state: "at_risk" | "breached";
|
||||
severity: "warning" | "critical";
|
||||
message: string;
|
||||
}
|
||||
|
||||
export const levelKey = (f: Pick<LevelFinding, "levelId">) => `slo:${f.levelId}`;
|
||||
|
||||
/** The targets whose budget is mostly or wholly used this month, for the ones that are to alert. */
|
||||
export async function levelFindings(now: Date = new Date()): Promise<LevelFinding[]> {
|
||||
const out: LevelFinding[] = [];
|
||||
for (const s of await levelStatuses(now, { history: false })) {
|
||||
if (!s.alert || (s.budget.state !== "at_risk" && s.budget.state !== "breached")) continue;
|
||||
out.push({ levelId: s.id, name: s.name, state: s.budget.state, severity: s.budget.state === "breached" ? "critical" : "warning", message: budgetMessage(s.name, s.target, s.period.label, s.budget, s.measure) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface PeriodResult {
|
||||
id: number;
|
||||
name: string;
|
||||
subjectType: SubjectType;
|
||||
target: number;
|
||||
percent: number | null;
|
||||
downMinutes: number;
|
||||
/** Over the whole period (even one that is not over yet). */
|
||||
allowedMinutes: number;
|
||||
usedPercent: number | null;
|
||||
/** Whether it stayed within its budget; null without a figure. For a period not over, "so far". */
|
||||
met: boolean | null;
|
||||
}
|
||||
|
||||
/** How each target did over a period (a month, a quarter, a year): for the reports. */
|
||||
export async function levelResultsFor(start: Date, end: Date, now: Date = new Date()): Promise<PeriodResult[]> {
|
||||
const rows = await listLevels();
|
||||
const ctx = newContext(now);
|
||||
const total = (end.getTime() - start.getTime()) / 60_000;
|
||||
const out: PeriodResult[] = [];
|
||||
for (const r of rows) {
|
||||
const m = await measure(r.subjectType as SubjectType, r.subjectId, start, end, ctx);
|
||||
if (m.percent === null) continue;
|
||||
const b = judgeBudget(r.target, total, (Math.min(now.getTime(), end.getTime()) - start.getTime()) / 60_000, m);
|
||||
out.push({ id: r.id, name: r.name, subjectType: r.subjectType as SubjectType, target: r.target, percent: m.percent, downMinutes: m.downMinutes, allowedMinutes: Math.round(b.allowedMinutes * 10) / 10, usedPercent: b.usedPercent, met: m.downMinutes <= b.allowedMinutes });
|
||||
}
|
||||
return out.sort((a, b) => Number(a.met) - Number(b.met) || (b.usedPercent ?? 0) - (a.usedPercent ?? 0) || a.name.localeCompare(b.name));
|
||||
}
|
||||
@@ -128,6 +128,10 @@ export interface HealthCheckSettings {
|
||||
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
|
||||
export interface MonthlyReportSettings {
|
||||
enabled: boolean;
|
||||
/** Also send the quarter's report after the month that ends it (March, June, September, December). */
|
||||
quarterly: boolean;
|
||||
/** Also send the year's report after December. */
|
||||
annual: boolean;
|
||||
/** Day of the month, 1–28 (every month has it), in notifications.timezone. */
|
||||
day: number;
|
||||
/** "HH:MM", in notifications.timezone. */
|
||||
@@ -293,7 +297,7 @@ const DEFAULTS: AppSettings = {
|
||||
logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 },
|
||||
quietHours: { enabled: false, start: "22:00", end: "07:00" },
|
||||
digest: { enabled: false, day: 1, time: "08:00" },
|
||||
monthlyReport: { enabled: false, day: 1, time: "08:00" },
|
||||
monthlyReport: { enabled: false, quarterly: false, annual: false, day: 1, time: "08:00" },
|
||||
inventory: { currency: "", pricePerKwh: null },
|
||||
agents: { autoUpdate: false },
|
||||
alertActions: { enabled: false },
|
||||
|
||||
@@ -34,13 +34,17 @@ export interface UptimeSeries {
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface Segment {
|
||||
from: number;
|
||||
to: number;
|
||||
down: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits [knownFrom, now] into up and down stretches from the recorded transitions and lays them over `days` consecutive 24-hour
|
||||
* buckets ending at `now`. Before the first transition the thing is taken to have been in the opposite state (a "went offline" means it
|
||||
* was up until then); with no transitions at all it has been up the whole time.
|
||||
* Splits [knownFrom, now] into up and down stretches from the recorded transitions. Before the first transition the thing is taken to have been in the opposite state
|
||||
* (a "went offline" means it was up until then); with no transitions at all it has been up the whole time.
|
||||
*/
|
||||
export function computeBuckets(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeBucket[] {
|
||||
const start = now - days * DAY_MS;
|
||||
export function segmentsOf(transitions: Transition[], knownFrom: number, now: number): Segment[] {
|
||||
const sorted = [...transitions].sort((a, b) => a.at - b.at);
|
||||
|
||||
// State at knownFrom: after any transitions that happened before it, otherwise the opposite of the first one that follows.
|
||||
@@ -49,7 +53,7 @@ export function computeBuckets(transitions: Transition[], knownFrom: number, now
|
||||
while (i < sorted.length && sorted[i].at <= knownFrom) state = sorted[i++].down;
|
||||
if (i === 0 && sorted.length > 0) state = !sorted[0].down;
|
||||
|
||||
const segments: { from: number; to: number; down: boolean }[] = [];
|
||||
const segments: Segment[] = [];
|
||||
let cursor = Math.min(knownFrom, now);
|
||||
for (; i < sorted.length; i++) {
|
||||
const t = Math.min(sorted[i].at, now);
|
||||
@@ -58,6 +62,26 @@ export function computeBuckets(transitions: Transition[], knownFrom: number, now
|
||||
state = sorted[i].down;
|
||||
}
|
||||
if (now > cursor) segments.push({ from: cursor, to: now, down: state });
|
||||
return segments;
|
||||
}
|
||||
|
||||
/** Minutes known, and of those spent down, within [from, to]. */
|
||||
export function minutesWithin(segments: Segment[], from: number, to: number): { known: number; down: number } {
|
||||
let known = 0;
|
||||
let down = 0;
|
||||
for (const seg of segments) {
|
||||
const overlap = Math.min(seg.to, to) - Math.max(seg.from, from);
|
||||
if (overlap <= 0) continue;
|
||||
known += overlap;
|
||||
if (seg.down) down += overlap;
|
||||
}
|
||||
return { known: Math.round(known / 60_000), down: Math.round(down / 60_000) };
|
||||
}
|
||||
|
||||
/** The segments laid over `days` consecutive 24-hour buckets ending at `now`. */
|
||||
export function computeBuckets(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeBucket[] {
|
||||
const start = now - days * DAY_MS;
|
||||
const segments = segmentsOf(transitions, knownFrom, now);
|
||||
|
||||
const known = new Array<number>(days).fill(0);
|
||||
const down = new Array<number>(days).fill(0);
|
||||
@@ -121,10 +145,49 @@ export interface UptimeReport {
|
||||
integrations: Record<number, UptimeSeries>;
|
||||
}
|
||||
|
||||
export async function getUptime(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<UptimeReport> {
|
||||
const nowMs = now.getTime();
|
||||
const rows = await db.select().from(auditLog).where(inArray(auditLog.action, ["went_offline", "back_online", "went_failing", "recovered"]));
|
||||
/** How long a server or integration was known, and down, between two moments. */
|
||||
export interface UptimeTotals {
|
||||
known: number;
|
||||
down: number;
|
||||
}
|
||||
|
||||
export interface UptimeRange {
|
||||
servers: Record<number, UptimeTotals>;
|
||||
integrations: Record<number, UptimeTotals>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Exactly how long each server and integration was down between `from` and `to` (the end is clipped to now), to the minute — for a target that must be measured over a
|
||||
* calendar period rather than in whole days back from now. Only the ones known for some of it are listed.
|
||||
*/
|
||||
export async function getUptimeRange(from: Date, to: Date, now: Date = new Date()): Promise<UptimeRange> {
|
||||
const nowMs = now.getTime();
|
||||
const end = Math.min(to.getTime(), nowMs);
|
||||
const start = from.getTime();
|
||||
const out: UptimeRange = { servers: {}, integrations: {} };
|
||||
if (end <= start) return out;
|
||||
const { serverEvents, sourceEvents } = await loadTransitions(nowMs);
|
||||
|
||||
for (const s of await db.select({ id: servers.id, createdAt: servers.createdAt, lastSeenAt: servers.lastSeenAt }).from(servers)) {
|
||||
const events = serverEvents.get(s.id) ?? [];
|
||||
if (!s.lastSeenAt && events.length === 0) continue;
|
||||
const created = Date.parse(auditTimeToIso(s.createdAt));
|
||||
const totals = minutesWithin(segmentsOf(events, Number.isFinite(created) ? created : nowMs, nowMs), start, end);
|
||||
if (totals.known > 0) out.servers[s.id] = totals;
|
||||
}
|
||||
const since = await ensureHistoryStart(nowMs);
|
||||
for (const i of await db.select({ id: integrations.id, type: integrations.type, createdAt: integrations.createdAt }).from(integrations)) {
|
||||
const created = Date.parse(auditTimeToIso(i.createdAt));
|
||||
const knownFrom = Math.max(Number.isFinite(created) ? created : 0, since);
|
||||
const totals = minutesWithin(segmentsOf(sourceEvents.get(i.type) ?? [], knownFrom, nowMs), start, end);
|
||||
if (totals.known > 0) out.integrations[i.id] = totals;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The recorded moments servers went away and came back, and integrations started failing and worked again. */
|
||||
async function loadTransitions(nowMs: number): Promise<{ serverEvents: Map<number, Transition[]>; sourceEvents: Map<string, Transition[]> }> {
|
||||
const rows = await db.select().from(auditLog).where(inArray(auditLog.action, ["went_offline", "back_online", "went_failing", "recovered"]));
|
||||
const serverEvents = new Map<number, Transition[]>();
|
||||
const sourceEvents = new Map<string, Transition[]>();
|
||||
for (const r of rows) {
|
||||
@@ -141,6 +204,12 @@ export async function getUptime(now: Date = new Date(), days: number = DEFAULT_D
|
||||
sourceEvents.set(r.targetId, list);
|
||||
}
|
||||
}
|
||||
return { serverEvents, sourceEvents };
|
||||
}
|
||||
|
||||
export async function getUptime(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<UptimeReport> {
|
||||
const nowMs = now.getTime();
|
||||
const { serverEvents, sourceEvents } = await loadTransitions(nowMs);
|
||||
|
||||
const out: UptimeReport = { days, endsAt: now.toISOString(), servers: {}, integrations: {} };
|
||||
|
||||
|
||||
Reference in new issue
Block a user