Files
Homelab-manager/server/src/services/monthlyReport.ts
T
bobbanandClaude Sonnet 5.5 3b31b16796 Service-level targets with error budgets, and quarterly and annual reports
Monitoring -> Service levels: a target (99.9 %) on an endpoint check, a server or an
integration, measured over calendar months. Shows the figure, downtime used, the
error budget left, and how the last six months went. 80 % of the budget used is a
warning and an overspent budget a critical alert. Endpoints are measured day by day,
servers and integrations to the minute from the audit log.

The report page gains Quarter and Year (same figures, with a Service levels section),
and two settings send the quarterly and annual report after the month that ends them.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-21 22:20:07 +02:00

386 lines
21 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* The monthly report (and, by the same figures over longer stretches, the quarterly and annual ones): how a month went — availability of servers, integrations and endpoints, incidents and how long they took, updates applied (by hand and by
* patch plans), restore tests and the state of the backups, and what was renewed. Everything in it the app already knows; this gathers it for one calendar month
* (on the clock of the notification time zone) into one structure that the page shows, prints and exports, and into a plain-text version that can be sent like the
* weekly digest.
*
* What it can't say, it says: servers' and integrations' availability only goes back as far as the audit log records it; endpoint availability from the daily
* figures (kept since they were added, and started from the last two weeks of results); renewals only from when the app began noticing them; the backups are the
* state now, not a history. A month with nothing to report for a part leaves that part out.
*/
import { and, gte, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, endpointDaily, integrations, patchPlanRuns, restoreTests, servers, updateRuns } from "../db/schema.js";
import { getCachedBackupCoverage } from "./backupCoverage.js";
import { listIncidents } from "./incidents.js";
import { zonedTime } from "./maintenanceSchedules.js";
import { renewalsBetween } from "./renewals.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
import { spanWords } from "./weeklyDigest.js";
import { levelResultsFor, type PeriodResult } from "./serviceLevels.js";
const DAY_MS = 86_400_000;
const MAX_LISTED = 12;
export interface AvailabilityRow {
id: number;
name: string;
/** Share of the known time spent up, in percent; null when nothing is known for the month. */
percent: number | null;
downMinutes: number;
}
export interface EndpointRow {
id: number;
name: string;
percent: number | null;
avgMs: number | null;
checks: number;
}
export type PeriodKind = "month" | "quarter" | "year";
export interface MonthlyReport {
/** What stretch it is about: a month (id "2026-09"), a quarter ("2026-Q3") or a year ("2026"). */
period: PeriodKind;
/** The id of that stretch. (Named for the month, which is what it was made for.) */
month: string;
/** "September 2026", "Q3 2026", "2026". */
label: string;
/** ISO — the start of the month and its end (or now, for a month that is not over). */
from: string;
to: string;
/** The month is not over yet: everything is "so far". */
partial: boolean;
timezone: string;
servers: AvailabilityRow[];
integrations: AvailabilityRow[];
endpoints: EndpointRow[];
/** How each service-level target did over the stretch (those with a figure for it). */
serviceLevels: PeriodResult[];
incidents: {
began: number;
resolved: number;
stillOpen: number;
/** Of those that began and ended in the month. */
totalMinutes: number;
averageMinutes: number | null;
longest: { id: number; message: string; minutes: number } | null;
byCategory: { category: string; count: number }[];
/** The longest, up to a dozen. */
list: { id: number; message: string; where: string; severity: string; openedAt: string; resolvedAt: string | null; minutes: number }[];
};
updates: {
runs: number;
done: number;
failed: number;
/** Updates the runs say they installed (before minus after, where both are known). */
installed: number;
byServer: { name: string; runs: number; failed: number; installed: number }[];
planRuns: { runs: number; done: number; failed: number };
};
restoreTests: { label: string; testedAt: string; result: string; note: string | null }[];
/** The backup state as of now; null when no coverage has been read yet. */
backups: { ok: number; stale: number; failed: number; none: number; unknown: number; total: number; asOf: string } | null;
renewals: { kind: string; subject: string; at: string; fromDate: string; toDate: string }[];
}
// ─── Months ────────────────────────────────────────────────────────────────
const pad = (n: number) => String(n).padStart(2, "0");
export function isMonthId(v: unknown): v is string {
return typeof v === "string" && /^\d{4}-(0[1-9]|1[0-2])$/.test(v);
}
/** The year and month `now` is in, on the clock of `tz`. */
function monthOf(now: Date, tz: string): { year: number; month: number } {
const parts = new Intl.DateTimeFormat("en-CA", { timeZone: tz, year: "numeric", month: "2-digit" }).formatToParts(now);
return { year: Number(parts.find((p) => p.type === "year")?.value), month: Number(parts.find((p) => p.type === "month")?.value) };
}
export const monthId = (year: number, month: number) => `${year}-${pad(month)}`;
export function isQuarterId(v: unknown): boolean {
return typeof v === "string" && /^\d{4}-Q[1-4]$/.test(v);
}
export function isYearId(v: unknown): boolean {
return typeof v === "string" && /^\d{4}$/.test(v);
}
export const isPeriodId = (v: unknown): v is string => isMonthId(v) || isQuarterId(v) || isYearId(v);
export function periodKind(id: string): PeriodKind {
return isQuarterId(id) ? "quarter" : isYearId(id) ? "year" : "month";
}
const shift = (year: number, month: number, by: number) => {
const index = year * 12 + (month - 1) + by;
return { year: Math.floor(index / 12), month: (index % 12) + 1 };
};
export function monthLabel(id: string): string {
const [y, m] = id.split("-").map(Number);
return new Date(Date.UTC(y, m - 1, 15)).toLocaleDateString("en-GB", { month: "long", year: "numeric", timeZone: "UTC" });
}
/** The month before the one `now` is in: what a report sent at the start of a month is about. */
export function lastFullMonth(now: Date, tz: string): string {
const cur = monthOf(now, tz);
const prev = shift(cur.year, cur.month, -1);
return monthId(prev.year, prev.month);
}
/** This month and the eleven before it, newest first. */
export function recentMonths(now: Date, tz: string, count = 12): { id: string; label: string }[] {
const cur = monthOf(now, tz);
return Array.from({ length: count }, (_, i) => {
const m = shift(cur.year, cur.month, -i);
const id = monthId(m.year, m.month);
return { id, label: monthLabel(id) };
});
}
export function periodLabel(id: string): string {
if (isQuarterId(id)) return `Q${id.slice(6)} ${id.slice(0, 4)}`;
return isYearId(id) ? id : monthLabel(id);
}
/** The year and quarter (1–4) `now` is in. */
function quarterOf(now: Date, tz: string): { year: number; quarter: number } {
const { year, month } = monthOf(now, tz);
return { year, quarter: Math.ceil(month / 3) };
}
/** The last whole quarter ("2026-Q2" in July), and the last whole year. */
export function lastFullQuarter(now: Date, tz: string): string {
const { year, quarter } = quarterOf(now, tz);
return quarter === 1 ? `${year - 1}-Q4` : `${year}-Q${quarter - 1}`;
}
export const lastFullYear = (now: Date, tz: string) => String(monthOf(now, tz).year - 1);
/** What can be asked for: this month and eleven before it, this quarter and seven before it, this year and two before it — newest first. */
export function recentPeriods(now: Date, tz: string): { months: { id: string; label: string }[]; quarters: { id: string; label: string }[]; years: { id: string; label: string }[] } {
const { year, quarter } = quarterOf(now, tz);
const quarters = Array.from({ length: 8 }, (_, i) => {
const index = year * 4 + (quarter - 1) - i;
const id = `${Math.floor(index / 4)}-Q${(index % 4) + 1}`;
return { id, label: periodLabel(id) };
});
const years = Array.from({ length: 3 }, (_, i) => ({ id: String(year - i), label: String(year - i) }));
return { months: recentMonths(now, tz), quarters, years };
}
export interface PeriodBounds {
start: Date;
end: Date;
/** The first UTC day of it and of what follows, as "YYYY-MM-DD", for the figures kept per UTC day. */
firstDay: string;
nextFirstDay: string;
}
/** The instants a month, quarter or year begins and ends at, on the clock of `tz`. */
export function periodBounds(id: string, tz: string): PeriodBounds {
if (isYearId(id)) {
const y = Number(id);
return { start: zonedTime(y, 1, 1, 0, 0, tz), end: zonedTime(y + 1, 1, 1, 0, 0, tz), firstDay: `${y}-01-01`, nextFirstDay: `${y + 1}-01-01` };
}
if (isQuarterId(id)) {
const y = Number(id.slice(0, 4));
const first = (Number(id.slice(6)) - 1) * 3 + 1;
const next = shift(y, first, 3);
return { start: zonedTime(y, first, 1, 0, 0, tz), end: zonedTime(next.year, next.month, 1, 0, 0, tz), firstDay: `${y}-${pad(first)}-01`, nextFirstDay: `${monthId(next.year, next.month)}-01` };
}
const [y, m] = id.split("-").map(Number);
const next = shift(y, m, 1);
return { ...monthBounds(id, tz), firstDay: `${id}-01`, nextFirstDay: `${monthId(next.year, next.month)}-01` };
}
/** The instants a month begins and ends at, on the clock of `tz`. */
export function monthBounds(id: string, tz: string): { start: Date; end: Date } {
const [y, m] = id.split("-").map(Number);
const next = shift(y, m, 1);
return { start: zonedTime(y, m, 1, 0, 0, tz), end: zonedTime(next.year, next.month, 1, 0, 0, tz) };
}
// ─── Gathering ─────────────────────────────────────────────────────────────
const round1 = (n: number) => Math.round(n * 10) / 10;
export async function buildMonthlyReport(id: string, now: Date = new Date()): Promise<MonthlyReport> {
const settings = await getSettings();
const tz = settings.notifications.timezone || "UTC";
const bounds = periodBounds(id, tz);
const { start, end: fullEnd } = bounds;
const partial = fullEnd.getTime() > now.getTime();
const end = partial ? now : fullEnd;
const days = Math.max(1, Math.ceil((end.getTime() - start.getTime()) / DAY_MS));
const startIso = start.toISOString();
const endIso = end.toISOString();
// Availability. Servers and integrations from the audit log's record of going away and coming back; laid over the month's days ending where the month does.
const uptime = await getUptime(end, days);
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
const integrationNames = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name]));
const rowsOf = (series: Record<number, { buckets: { k: number; d: number }[]; percent: number }>, names: Map<number, string>): AvailabilityRow[] =>
Object.entries(series)
.map(([key, s]) => ({ id: Number(key), name: names.get(Number(key)) ?? `#${key}`, percent: s.percent, downMinutes: s.buckets.reduce((n, b) => n + b.d, 0) }))
.filter((r) => names.has(r.id))
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
// Endpoints: from the daily figures, for the days the month covers. The figures are per UTC day, so a day at either end can belong to a neighbouring month
// by a few hours (more than the time zone is ahead of UTC); a month that is not over takes in today.
const firstDay = bounds.firstDay;
const lastDayExclusive = partial ? new Date(now.getTime() + DAY_MS).toISOString().slice(0, 10) : bounds.nextFirstDay;
const dailyRows = await db.select().from(endpointDaily).where(and(gte(endpointDaily.day, firstDay), lt(endpointDaily.day, lastDayExclusive)));
const checkNames = new Map((await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks)).map((c) => [c.id, c.name]));
const perCheck = new Map<number, { total: number; ok: number; msSum: number; msCount: number }>();
for (const r of dailyRows) {
const t = perCheck.get(r.checkId) ?? { total: 0, ok: 0, msSum: 0, msCount: 0 };
t.total += r.total;
t.ok += r.ok;
t.msSum += r.msSum;
t.msCount += r.msCount;
perCheck.set(r.checkId, t);
}
const endpointRows: EndpointRow[] = [...perCheck]
.filter(([cid, t]) => checkNames.has(cid) && t.total > 0)
.map(([cid, t]) => ({ id: cid, name: checkNames.get(cid)!, percent: round1((t.ok / t.total) * 100), avgMs: t.msCount > 0 ? Math.round(t.msSum / t.msCount) : null, checks: t.total }))
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
// Incidents.
const all = await listIncidents({ now: end, limit: 20000 });
const inMonth = (iso: string) => iso >= startIso && iso < endIso;
const began = all.filter((i) => inMonth(i.openedAt));
const resolved = all.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
const stillOpen = all.filter((i) => i.openedAt < endIso && (i.resolvedAt === null || i.resolvedAt >= endIso)).length;
const closedHere = began.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
const minutesOf = (ms: number) => Math.max(1, Math.round(ms / 60_000));
const totalMinutes = closedHere.reduce((n, i) => n + minutesOf(i.durationMs), 0);
const byDuration = [...began].sort((a, b) => b.durationMs - a.durationMs);
const categories = new Map<string, number>();
for (const i of began) categories.set(i.category, (categories.get(i.category) ?? 0) + 1);
// Updates, by hand and by plan.
const runRows = await db.select().from(updateRuns).where(and(gte(updateRuns.startedAt, startIso), lt(updateRuns.startedAt, endIso)));
const perServer = new Map<number, { runs: number; failed: number; installed: number }>();
let installed = 0;
for (const r of runRows) {
const gained = r.status === "done" && r.updatesBefore !== null && r.updatesAfter !== null ? Math.max(0, r.updatesBefore - r.updatesAfter) : 0;
installed += gained;
const t = perServer.get(r.serverId) ?? { runs: 0, failed: 0, installed: 0 };
t.runs++;
if (r.status === "failed") t.failed++;
t.installed += gained;
perServer.set(r.serverId, t);
}
const planRows = await db.select().from(patchPlanRuns).where(and(gte(patchPlanRuns.startedAt, startIso), lt(patchPlanRuns.startedAt, endIso)));
// Restore tests, and the backups as they are now.
const tests = await db.select().from(restoreTests).where(and(gte(restoreTests.testedAt, firstDay), lt(restoreTests.testedAt, lastDayExclusive)));
const coverage = await getCachedBackupCoverage(now);
const renewalRows = await renewalsBetween(start, end);
return {
period: periodKind(id),
month: id,
label: periodLabel(id),
from: startIso,
to: endIso,
partial,
timezone: tz,
servers: rowsOf(uptime.servers, serverNames),
integrations: rowsOf(uptime.integrations, integrationNames),
endpoints: endpointRows,
serviceLevels: await levelResultsFor(start, fullEnd, now),
incidents: {
began: began.length,
resolved: resolved.length,
stillOpen,
totalMinutes,
averageMinutes: closedHere.length > 0 ? Math.round(totalMinutes / closedHere.length) : null,
longest: byDuration[0] ? { id: byDuration[0].id, message: byDuration[0].message, minutes: minutesOf(byDuration[0].durationMs) } : null,
byCategory: [...categories].map(([category, count]) => ({ category, count })).sort((a, b) => b.count - a.count || a.category.localeCompare(b.category)),
list: byDuration.slice(0, MAX_LISTED).map((i) => ({ id: i.id, message: i.message, where: i.target?.name ?? i.source, severity: i.peakSeverity, openedAt: i.openedAt, resolvedAt: i.resolvedAt, minutes: minutesOf(i.durationMs) })),
},
updates: {
runs: runRows.length,
done: runRows.filter((r) => r.status === "done").length,
failed: runRows.filter((r) => r.status === "failed").length,
installed,
byServer: [...perServer]
.map(([sid, t]) => ({ name: serverNames.get(sid) ?? `#${sid}`, ...t }))
.sort((a, b) => b.runs - a.runs || a.name.localeCompare(b.name)),
planRuns: { runs: planRows.length, done: planRows.filter((r) => r.status === "done").length, failed: planRows.filter((r) => r.status === "failed").length },
},
restoreTests: tests.map((t) => ({ label: t.label, testedAt: t.testedAt, result: t.result, note: t.note })).sort((a, b) => a.testedAt.localeCompare(b.testedAt)),
backups: coverage ? { ok: coverage.counts.ok, stale: coverage.counts.stale, failed: coverage.counts.failed, none: coverage.counts.none, unknown: coverage.counts.unknown, total: coverage.counts.total, asOf: coverage.generatedAt } : null,
renewals: renewalRows.map((r) => ({ kind: r.kind, subject: r.subject, at: r.at, fromDate: r.fromDate, toDate: r.toDate })),
};
}
// ─── As a message ──────────────────────────────────────────────────────────
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
const RESULT_WORDS: Record<string, string> = { ok: "passed", partial: "partly worked", failed: "failed", not_needed: "marked as not needed" };
function average(rows: AvailabilityRow[]): number | null {
const known = rows.filter((r) => r.percent !== null);
return known.length === 0 ? null : round1(known.reduce((n, r) => n + (r.percent as number), 0) / known.length);
}
/** The report as plain text, for a notification or an email: short, a topic only where there is something to say. */
export function monthlyReportText(r: MonthlyReport): { title: string; message: string } {
const sections: string[] = [];
const lines = (heading: string, body: string[]) => body.length > 0 && sections.push(`${heading.toUpperCase()}\n${body.join("\n")}`);
const avail: string[] = [];
const serverAvg = average(r.servers);
if (serverAvg !== null) {
avail.push(`Servers: ${serverAvg}% on average (${plural(r.servers.length, "server")}).`);
for (const s of r.servers.filter((x) => x.downMinutes > 0).slice(0, 6)) avail.push(`- ${s.name}: ${s.percent}% (${spanWords(s.downMinutes)} down)`);
}
if (r.endpoints.length > 0) {
const avg = round1(r.endpoints.reduce((n, e) => n + (e.percent ?? 0), 0) / r.endpoints.length);
avail.push(`Endpoints: ${avg}% on average (${plural(r.endpoints.length, "check")}).`);
for (const e of r.endpoints.filter((x) => (x.percent ?? 100) < 100).slice(0, 6)) avail.push(`- ${e.name}: ${e.percent}%${e.avgMs !== null ? `, ${e.avgMs} ms on average` : ""}`);
}
const integrationAvg = average(r.integrations);
if (integrationAvg !== null) avail.push(`Integrations: ${integrationAvg}% on average.`);
lines("Availability", avail);
if (r.serviceLevels.length > 0) {
const missed = r.serviceLevels.filter((l) => !l.met);
const body = [`${r.serviceLevels.length - missed.length} of ${plural(r.serviceLevels.length, "target")} met${r.partial ? " so far" : ""}.`];
for (const l of r.serviceLevels.slice(0, 8)) body.push(`- ${l.name}: ${l.percent}% against ${l.target}% — ${l.met ? "met" : "MISSED"} (down ${l.downMinutes > 0 ? spanWords(l.downMinutes) : "not at all"}, ${l.allowedMinutes >= 1 ? spanWords(l.allowedMinutes) : "under a minute"} allowed)`);
lines("Service levels", body);
}
const inc = r.incidents;
if (inc.began > 0 || inc.resolved > 0 || inc.stillOpen > 0) {
const body = [`${plural(inc.began, "incident")} began, ${inc.resolved} ended, ${inc.stillOpen} still open.`];
if (inc.averageMinutes !== null) body.push(`A resolved incident took ${spanWords(inc.averageMinutes)} on average.`);
for (const i of inc.list.slice(0, 3)) body.push(`- ${i.message} — ${spanWords(i.minutes)}${i.resolvedAt ? "" : " and still open"}`);
lines("Incidents", body);
} else if (serverAvg !== null || r.endpoints.length > 0) {
lines("Incidents", ["None."]);
}
const u = r.updates;
const upd: string[] = [];
if (u.runs > 0) upd.push(`${plural(u.runs, "update run")}: ${u.done} done${u.failed > 0 ? `, ${u.failed} failed` : ""}${u.installed > 0 ? ` — ${plural(u.installed, "update")} installed` : ""}.`);
if (u.planRuns.runs > 0) upd.push(`Patch plans ran ${plural(u.planRuns.runs, "time")}${u.planRuns.failed > 0 ? `, ${u.planRuns.failed} not cleanly` : ", all cleanly"}.`);
lines("Updates", upd);
const bk: string[] = [];
for (const t of r.restoreTests) bk.push(`- Restore test of ${t.label} (${t.testedAt}) ${RESULT_WORDS[t.result] ?? t.result}`);
if (r.restoreTests.length === 0) bk.push("No restore test was recorded.");
if (r.backups) bk.push(`Backups now: ${r.backups.ok} of ${plural(r.backups.total, "guest")} recent${r.backups.failed + r.backups.stale + r.backups.none > 0 ? `; ${r.backups.failed} failed, ${r.backups.stale} stale, ${r.backups.none} without a backup` : ""}.`);
lines("Backups", bk);
lines("Renewed", r.renewals.slice(0, 10).map((x) => `- ${x.kind === "domain" ? "Domain" : "Certificate"} ${x.subject}: ${x.fromDate} → ${x.toDate}`));
const noun = r.period === "year" ? "annual" : r.period === "quarter" ? "quarterly" : "monthly";
const title = `Homelab ${noun} report — ${r.label}${r.partial ? " (so far)" : ""}`;
return { title, message: sections.length > 0 ? sections.join("\n\n") : `Nothing was recorded for this ${r.period}.` };
}