/** * The monthly report (and, by the same figures over longer stretches, the quarterly and annual ones): how a month went — availability of servers, integrations and endpoints, incidents and how long they took, updates applied (by hand and by * patch plans), restore tests and the state of the backups, and what was renewed. Everything in it the app already knows; this gathers it for one calendar month * (on the clock of the notification time zone) into one structure that the page shows, prints and exports, and into a plain-text version that can be sent like the * weekly digest. * * What it can't say, it says: servers' and integrations' availability only goes back as far as the audit log records it; endpoint availability from the daily * figures (kept since they were added, and started from the last two weeks of results); renewals only from when the app began noticing them; the backups are the * state now, not a history. A month with nothing to report for a part leaves that part out. */ import { and, gte, lt } from "drizzle-orm"; import { db } from "../db/client.js"; import { endpointChecks, endpointDaily, integrations, patchPlanRuns, restoreTests, servers, updateRuns } from "../db/schema.js"; import { getCachedBackupCoverage } from "./backupCoverage.js"; import { listIncidents } from "./incidents.js"; import { zonedTime } from "./maintenanceSchedules.js"; import { renewalsBetween } from "./renewals.js"; import { getSettings } from "./settingsStore.js"; import { getUptime } from "./uptimeHistory.js"; import { spanWords } from "./weeklyDigest.js"; import { levelResultsFor, type PeriodResult } from "./serviceLevels.js"; const DAY_MS = 86_400_000; const MAX_LISTED = 12; export interface AvailabilityRow { id: number; name: string; /** Share of the known time spent up, in percent; null when nothing is known for the month. */ percent: number | null; downMinutes: number; } export interface EndpointRow { id: number; name: string; percent: number | null; avgMs: number | null; checks: number; } export type PeriodKind = "month" | "quarter" | "year"; export interface MonthlyReport { /** What stretch it is about: a month (id "2026-09"), a quarter ("2026-Q3") or a year ("2026"). */ period: PeriodKind; /** The id of that stretch. (Named for the month, which is what it was made for.) */ month: string; /** "September 2026", "Q3 2026", "2026". */ label: string; /** ISO — the start of the month and its end (or now, for a month that is not over). */ from: string; to: string; /** The month is not over yet: everything is "so far". */ partial: boolean; timezone: string; servers: AvailabilityRow[]; integrations: AvailabilityRow[]; endpoints: EndpointRow[]; /** How each service-level target did over the stretch (those with a figure for it). */ serviceLevels: PeriodResult[]; incidents: { began: number; resolved: number; stillOpen: number; /** Of those that began and ended in the month. */ totalMinutes: number; averageMinutes: number | null; longest: { id: number; message: string; minutes: number } | null; byCategory: { category: string; count: number }[]; /** The longest, up to a dozen. */ list: { id: number; message: string; where: string; severity: string; openedAt: string; resolvedAt: string | null; minutes: number }[]; }; updates: { runs: number; done: number; failed: number; /** Updates the runs say they installed (before minus after, where both are known). */ installed: number; byServer: { name: string; runs: number; failed: number; installed: number }[]; planRuns: { runs: number; done: number; failed: number }; }; restoreTests: { label: string; testedAt: string; result: string; note: string | null }[]; /** The backup state as of now; null when no coverage has been read yet. */ backups: { ok: number; stale: number; failed: number; none: number; unknown: number; total: number; asOf: string } | null; renewals: { kind: string; subject: string; at: string; fromDate: string; toDate: string }[]; } // ─── Months ──────────────────────────────────────────────────────────────── const pad = (n: number) => String(n).padStart(2, "0"); export function isMonthId(v: unknown): v is string { return typeof v === "string" && /^\d{4}-(0[1-9]|1[0-2])$/.test(v); } /** The year and month `now` is in, on the clock of `tz`. */ function monthOf(now: Date, tz: string): { year: number; month: number } { const parts = new Intl.DateTimeFormat("en-CA", { timeZone: tz, year: "numeric", month: "2-digit" }).formatToParts(now); return { year: Number(parts.find((p) => p.type === "year")?.value), month: Number(parts.find((p) => p.type === "month")?.value) }; } export const monthId = (year: number, month: number) => `${year}-${pad(month)}`; export function isQuarterId(v: unknown): boolean { return typeof v === "string" && /^\d{4}-Q[1-4]$/.test(v); } export function isYearId(v: unknown): boolean { return typeof v === "string" && /^\d{4}$/.test(v); } export const isPeriodId = (v: unknown): v is string => isMonthId(v) || isQuarterId(v) || isYearId(v); export function periodKind(id: string): PeriodKind { return isQuarterId(id) ? "quarter" : isYearId(id) ? "year" : "month"; } const shift = (year: number, month: number, by: number) => { const index = year * 12 + (month - 1) + by; return { year: Math.floor(index / 12), month: (index % 12) + 1 }; }; export function monthLabel(id: string): string { const [y, m] = id.split("-").map(Number); return new Date(Date.UTC(y, m - 1, 15)).toLocaleDateString("en-GB", { month: "long", year: "numeric", timeZone: "UTC" }); } /** The month before the one `now` is in: what a report sent at the start of a month is about. */ export function lastFullMonth(now: Date, tz: string): string { const cur = monthOf(now, tz); const prev = shift(cur.year, cur.month, -1); return monthId(prev.year, prev.month); } /** This month and the eleven before it, newest first. */ export function recentMonths(now: Date, tz: string, count = 12): { id: string; label: string }[] { const cur = monthOf(now, tz); return Array.from({ length: count }, (_, i) => { const m = shift(cur.year, cur.month, -i); const id = monthId(m.year, m.month); return { id, label: monthLabel(id) }; }); } export function periodLabel(id: string): string { if (isQuarterId(id)) return `Q${id.slice(6)} ${id.slice(0, 4)}`; return isYearId(id) ? id : monthLabel(id); } /** The year and quarter (1–4) `now` is in. */ function quarterOf(now: Date, tz: string): { year: number; quarter: number } { const { year, month } = monthOf(now, tz); return { year, quarter: Math.ceil(month / 3) }; } /** The last whole quarter ("2026-Q2" in July), and the last whole year. */ export function lastFullQuarter(now: Date, tz: string): string { const { year, quarter } = quarterOf(now, tz); return quarter === 1 ? `${year - 1}-Q4` : `${year}-Q${quarter - 1}`; } export const lastFullYear = (now: Date, tz: string) => String(monthOf(now, tz).year - 1); /** What can be asked for: this month and eleven before it, this quarter and seven before it, this year and two before it — newest first. */ export function recentPeriods(now: Date, tz: string): { months: { id: string; label: string }[]; quarters: { id: string; label: string }[]; years: { id: string; label: string }[] } { const { year, quarter } = quarterOf(now, tz); const quarters = Array.from({ length: 8 }, (_, i) => { const index = year * 4 + (quarter - 1) - i; const id = `${Math.floor(index / 4)}-Q${(index % 4) + 1}`; return { id, label: periodLabel(id) }; }); const years = Array.from({ length: 3 }, (_, i) => ({ id: String(year - i), label: String(year - i) })); return { months: recentMonths(now, tz), quarters, years }; } export interface PeriodBounds { start: Date; end: Date; /** The first UTC day of it and of what follows, as "YYYY-MM-DD", for the figures kept per UTC day. */ firstDay: string; nextFirstDay: string; } /** The instants a month, quarter or year begins and ends at, on the clock of `tz`. */ export function periodBounds(id: string, tz: string): PeriodBounds { if (isYearId(id)) { const y = Number(id); return { start: zonedTime(y, 1, 1, 0, 0, tz), end: zonedTime(y + 1, 1, 1, 0, 0, tz), firstDay: `${y}-01-01`, nextFirstDay: `${y + 1}-01-01` }; } if (isQuarterId(id)) { const y = Number(id.slice(0, 4)); const first = (Number(id.slice(6)) - 1) * 3 + 1; const next = shift(y, first, 3); return { start: zonedTime(y, first, 1, 0, 0, tz), end: zonedTime(next.year, next.month, 1, 0, 0, tz), firstDay: `${y}-${pad(first)}-01`, nextFirstDay: `${monthId(next.year, next.month)}-01` }; } const [y, m] = id.split("-").map(Number); const next = shift(y, m, 1); return { ...monthBounds(id, tz), firstDay: `${id}-01`, nextFirstDay: `${monthId(next.year, next.month)}-01` }; } /** The instants a month begins and ends at, on the clock of `tz`. */ export function monthBounds(id: string, tz: string): { start: Date; end: Date } { const [y, m] = id.split("-").map(Number); const next = shift(y, m, 1); return { start: zonedTime(y, m, 1, 0, 0, tz), end: zonedTime(next.year, next.month, 1, 0, 0, tz) }; } // ─── Gathering ───────────────────────────────────────────────────────────── const round1 = (n: number) => Math.round(n * 10) / 10; export async function buildMonthlyReport(id: string, now: Date = new Date()): Promise { const settings = await getSettings(); const tz = settings.notifications.timezone || "UTC"; const bounds = periodBounds(id, tz); const { start, end: fullEnd } = bounds; const partial = fullEnd.getTime() > now.getTime(); const end = partial ? now : fullEnd; const days = Math.max(1, Math.ceil((end.getTime() - start.getTime()) / DAY_MS)); const startIso = start.toISOString(); const endIso = end.toISOString(); // Availability. Servers and integrations from the audit log's record of going away and coming back; laid over the month's days ending where the month does. const uptime = await getUptime(end, days); const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name])); const integrationNames = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name])); const rowsOf = (series: Record, names: Map): AvailabilityRow[] => Object.entries(series) .map(([key, s]) => ({ id: Number(key), name: names.get(Number(key)) ?? `#${key}`, percent: s.percent, downMinutes: s.buckets.reduce((n, b) => n + b.d, 0) })) .filter((r) => names.has(r.id)) .sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name)); // Endpoints: from the daily figures, for the days the month covers. The figures are per UTC day, so a day at either end can belong to a neighbouring month // by a few hours (more than the time zone is ahead of UTC); a month that is not over takes in today. const firstDay = bounds.firstDay; const lastDayExclusive = partial ? new Date(now.getTime() + DAY_MS).toISOString().slice(0, 10) : bounds.nextFirstDay; const dailyRows = await db.select().from(endpointDaily).where(and(gte(endpointDaily.day, firstDay), lt(endpointDaily.day, lastDayExclusive))); const checkNames = new Map((await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks)).map((c) => [c.id, c.name])); const perCheck = new Map(); for (const r of dailyRows) { const t = perCheck.get(r.checkId) ?? { total: 0, ok: 0, msSum: 0, msCount: 0 }; t.total += r.total; t.ok += r.ok; t.msSum += r.msSum; t.msCount += r.msCount; perCheck.set(r.checkId, t); } const endpointRows: EndpointRow[] = [...perCheck] .filter(([cid, t]) => checkNames.has(cid) && t.total > 0) .map(([cid, t]) => ({ id: cid, name: checkNames.get(cid)!, percent: round1((t.ok / t.total) * 100), avgMs: t.msCount > 0 ? Math.round(t.msSum / t.msCount) : null, checks: t.total })) .sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name)); // Incidents. const all = await listIncidents({ now: end, limit: 20000 }); const inMonth = (iso: string) => iso >= startIso && iso < endIso; const began = all.filter((i) => inMonth(i.openedAt)); const resolved = all.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt)); const stillOpen = all.filter((i) => i.openedAt < endIso && (i.resolvedAt === null || i.resolvedAt >= endIso)).length; const closedHere = began.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt)); const minutesOf = (ms: number) => Math.max(1, Math.round(ms / 60_000)); const totalMinutes = closedHere.reduce((n, i) => n + minutesOf(i.durationMs), 0); const byDuration = [...began].sort((a, b) => b.durationMs - a.durationMs); const categories = new Map(); for (const i of began) categories.set(i.category, (categories.get(i.category) ?? 0) + 1); // Updates, by hand and by plan. const runRows = await db.select().from(updateRuns).where(and(gte(updateRuns.startedAt, startIso), lt(updateRuns.startedAt, endIso))); const perServer = new Map(); let installed = 0; for (const r of runRows) { const gained = r.status === "done" && r.updatesBefore !== null && r.updatesAfter !== null ? Math.max(0, r.updatesBefore - r.updatesAfter) : 0; installed += gained; const t = perServer.get(r.serverId) ?? { runs: 0, failed: 0, installed: 0 }; t.runs++; if (r.status === "failed") t.failed++; t.installed += gained; perServer.set(r.serverId, t); } const planRows = await db.select().from(patchPlanRuns).where(and(gte(patchPlanRuns.startedAt, startIso), lt(patchPlanRuns.startedAt, endIso))); // Restore tests, and the backups as they are now. const tests = await db.select().from(restoreTests).where(and(gte(restoreTests.testedAt, firstDay), lt(restoreTests.testedAt, lastDayExclusive))); const coverage = await getCachedBackupCoverage(now); const renewalRows = await renewalsBetween(start, end); return { period: periodKind(id), month: id, label: periodLabel(id), from: startIso, to: endIso, partial, timezone: tz, servers: rowsOf(uptime.servers, serverNames), integrations: rowsOf(uptime.integrations, integrationNames), endpoints: endpointRows, serviceLevels: await levelResultsFor(start, fullEnd, now), incidents: { began: began.length, resolved: resolved.length, stillOpen, totalMinutes, averageMinutes: closedHere.length > 0 ? Math.round(totalMinutes / closedHere.length) : null, longest: byDuration[0] ? { id: byDuration[0].id, message: byDuration[0].message, minutes: minutesOf(byDuration[0].durationMs) } : null, byCategory: [...categories].map(([category, count]) => ({ category, count })).sort((a, b) => b.count - a.count || a.category.localeCompare(b.category)), list: byDuration.slice(0, MAX_LISTED).map((i) => ({ id: i.id, message: i.message, where: i.target?.name ?? i.source, severity: i.peakSeverity, openedAt: i.openedAt, resolvedAt: i.resolvedAt, minutes: minutesOf(i.durationMs) })), }, updates: { runs: runRows.length, done: runRows.filter((r) => r.status === "done").length, failed: runRows.filter((r) => r.status === "failed").length, installed, byServer: [...perServer] .map(([sid, t]) => ({ name: serverNames.get(sid) ?? `#${sid}`, ...t })) .sort((a, b) => b.runs - a.runs || a.name.localeCompare(b.name)), planRuns: { runs: planRows.length, done: planRows.filter((r) => r.status === "done").length, failed: planRows.filter((r) => r.status === "failed").length }, }, restoreTests: tests.map((t) => ({ label: t.label, testedAt: t.testedAt, result: t.result, note: t.note })).sort((a, b) => a.testedAt.localeCompare(b.testedAt)), backups: coverage ? { ok: coverage.counts.ok, stale: coverage.counts.stale, failed: coverage.counts.failed, none: coverage.counts.none, unknown: coverage.counts.unknown, total: coverage.counts.total, asOf: coverage.generatedAt } : null, renewals: renewalRows.map((r) => ({ kind: r.kind, subject: r.subject, at: r.at, fromDate: r.fromDate, toDate: r.toDate })), }; } // ─── As a message ────────────────────────────────────────────────────────── const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`; const RESULT_WORDS: Record = { ok: "passed", partial: "partly worked", failed: "failed", not_needed: "marked as not needed" }; function average(rows: AvailabilityRow[]): number | null { const known = rows.filter((r) => r.percent !== null); return known.length === 0 ? null : round1(known.reduce((n, r) => n + (r.percent as number), 0) / known.length); } /** The report as plain text, for a notification or an email: short, a topic only where there is something to say. */ export function monthlyReportText(r: MonthlyReport): { title: string; message: string } { const sections: string[] = []; const lines = (heading: string, body: string[]) => body.length > 0 && sections.push(`${heading.toUpperCase()}\n${body.join("\n")}`); const avail: string[] = []; const serverAvg = average(r.servers); if (serverAvg !== null) { avail.push(`Servers: ${serverAvg}% on average (${plural(r.servers.length, "server")}).`); for (const s of r.servers.filter((x) => x.downMinutes > 0).slice(0, 6)) avail.push(`- ${s.name}: ${s.percent}% (${spanWords(s.downMinutes)} down)`); } if (r.endpoints.length > 0) { const avg = round1(r.endpoints.reduce((n, e) => n + (e.percent ?? 0), 0) / r.endpoints.length); avail.push(`Endpoints: ${avg}% on average (${plural(r.endpoints.length, "check")}).`); for (const e of r.endpoints.filter((x) => (x.percent ?? 100) < 100).slice(0, 6)) avail.push(`- ${e.name}: ${e.percent}%${e.avgMs !== null ? `, ${e.avgMs} ms on average` : ""}`); } const integrationAvg = average(r.integrations); if (integrationAvg !== null) avail.push(`Integrations: ${integrationAvg}% on average.`); lines("Availability", avail); if (r.serviceLevels.length > 0) { const missed = r.serviceLevels.filter((l) => !l.met); const body = [`${r.serviceLevels.length - missed.length} of ${plural(r.serviceLevels.length, "target")} met${r.partial ? " so far" : ""}.`]; for (const l of r.serviceLevels.slice(0, 8)) body.push(`- ${l.name}: ${l.percent}% against ${l.target}% — ${l.met ? "met" : "MISSED"} (down ${l.downMinutes > 0 ? spanWords(l.downMinutes) : "not at all"}, ${l.allowedMinutes >= 1 ? spanWords(l.allowedMinutes) : "under a minute"} allowed)`); lines("Service levels", body); } const inc = r.incidents; if (inc.began > 0 || inc.resolved > 0 || inc.stillOpen > 0) { const body = [`${plural(inc.began, "incident")} began, ${inc.resolved} ended, ${inc.stillOpen} still open.`]; if (inc.averageMinutes !== null) body.push(`A resolved incident took ${spanWords(inc.averageMinutes)} on average.`); for (const i of inc.list.slice(0, 3)) body.push(`- ${i.message} — ${spanWords(i.minutes)}${i.resolvedAt ? "" : " and still open"}`); lines("Incidents", body); } else if (serverAvg !== null || r.endpoints.length > 0) { lines("Incidents", ["None."]); } const u = r.updates; const upd: string[] = []; if (u.runs > 0) upd.push(`${plural(u.runs, "update run")}: ${u.done} done${u.failed > 0 ? `, ${u.failed} failed` : ""}${u.installed > 0 ? ` — ${plural(u.installed, "update")} installed` : ""}.`); if (u.planRuns.runs > 0) upd.push(`Patch plans ran ${plural(u.planRuns.runs, "time")}${u.planRuns.failed > 0 ? `, ${u.planRuns.failed} not cleanly` : ", all cleanly"}.`); lines("Updates", upd); const bk: string[] = []; for (const t of r.restoreTests) bk.push(`- Restore test of ${t.label} (${t.testedAt}) ${RESULT_WORDS[t.result] ?? t.result}`); if (r.restoreTests.length === 0) bk.push("No restore test was recorded."); if (r.backups) bk.push(`Backups now: ${r.backups.ok} of ${plural(r.backups.total, "guest")} recent${r.backups.failed + r.backups.stale + r.backups.none > 0 ? `; ${r.backups.failed} failed, ${r.backups.stale} stale, ${r.backups.none} without a backup` : ""}.`); lines("Backups", bk); lines("Renewed", r.renewals.slice(0, 10).map((x) => `- ${x.kind === "domain" ? "Domain" : "Certificate"} ${x.subject}: ${x.fromDate} → ${x.toDate}`)); const noun = r.period === "year" ? "annual" : r.period === "quarter" ? "quarterly" : "monthly"; const title = `Homelab ${noun} report — ${r.label}${r.partial ? " (so far)" : ""}`; return { title, message: sections.length > 0 ? sections.join("\n\n") : `Nothing was recorded for this ${r.period}.` }; }