Patch plans, a monthly report and new-port alerts

Patch plans (Updates -> Patch plans): updates on a schedule, one server at a time, each by an
ordinary update run, stopping at the first failure by default, silencing what depends on a
server while it is updated, with one message at the end.

Monthly report (Operations -> Monthly report, and optionally a message on a day of the month):
availability, incidents, updates, restore tests, backups and renewals for a calendar month,
printable. Backed by daily endpoint figures and a record of renewals.

New ports: the first report with ports is the baseline; a TCP port that begins listening later and
is not noted on the server's Ports card is a warning, cleared by Mark as expected.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 23:16:14 +02:00
1 parent ddd0aa622a
commit be6c5552bd
46 files changed
+7836 -26

No files matched your search

+303
View File
@@ -0,0 +1,303 @@
/**
* The monthly report: how a month went — availability of servers, integrations and endpoints, incidents and how long they took, updates applied (by hand and by
* patch plans), restore tests and the state of the backups, and what was renewed. Everything in it the app already knows; this gathers it for one calendar month
* (on the clock of the notification time zone) into one structure that the page shows, prints and exports, and into a plain-text version that can be sent like the
* weekly digest.
*
* What it can't say, it says: servers' and integrations' availability only goes back as far as the audit log records it; endpoint availability from the daily
* figures (kept since they were added, and started from the last two weeks of results); renewals only from when the app began noticing them; the backups are the
* state now, not a history. A month with nothing to report for a part leaves that part out.
*/
import { and, gte, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, endpointDaily, integrations, patchPlanRuns, restoreTests, servers, updateRuns } from "../db/schema.js";
import { getCachedBackupCoverage } from "./backupCoverage.js";
import { listIncidents } from "./incidents.js";
import { zonedTime } from "./maintenanceSchedules.js";
import { renewalsBetween } from "./renewals.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
import { spanWords } from "./weeklyDigest.js";
const DAY_MS = 86_400_000;
const MAX_LISTED = 12;
export interface AvailabilityRow {
id: number;
name: string;
/** Share of the known time spent up, in percent; null when nothing is known for the month. */
percent: number | null;
downMinutes: number;
}
export interface EndpointRow {
id: number;
name: string;
percent: number | null;
avgMs: number | null;
checks: number;
}
export interface MonthlyReport {
/** "2026-09". */
month: string;
/** "September 2026". */
label: string;
/** ISO — the start of the month and its end (or now, for a month that is not over). */
from: string;
to: string;
/** The month is not over yet: everything is "so far". */
partial: boolean;
timezone: string;
servers: AvailabilityRow[];
integrations: AvailabilityRow[];
endpoints: EndpointRow[];
incidents: {
began: number;
resolved: number;
stillOpen: number;
/** Of those that began and ended in the month. */
totalMinutes: number;
averageMinutes: number | null;
longest: { id: number; message: string; minutes: number } | null;
byCategory: { category: string; count: number }[];
/** The longest, up to a dozen. */
list: { id: number; message: string; where: string; severity: string; openedAt: string; resolvedAt: string | null; minutes: number }[];
};
updates: {
runs: number;
done: number;
failed: number;
/** Updates the runs say they installed (before minus after, where both are known). */
installed: number;
byServer: { name: string; runs: number; failed: number; installed: number }[];
planRuns: { runs: number; done: number; failed: number };
};
restoreTests: { label: string; testedAt: string; result: string; note: string | null }[];
/** The backup state as of now; null when no coverage has been read yet. */
backups: { ok: number; stale: number; failed: number; none: number; unknown: number; total: number; asOf: string } | null;
renewals: { kind: string; subject: string; at: string; fromDate: string; toDate: string }[];
}
// ─── Months ────────────────────────────────────────────────────────────────
const pad = (n: number) => String(n).padStart(2, "0");
export function isMonthId(v: unknown): v is string {
return typeof v === "string" && /^\d{4}-(0[1-9]|1[0-2])$/.test(v);
}
/** The year and month `now` is in, on the clock of `tz`. */
function monthOf(now: Date, tz: string): { year: number; month: number } {
const parts = new Intl.DateTimeFormat("en-CA", { timeZone: tz, year: "numeric", month: "2-digit" }).formatToParts(now);
return { year: Number(parts.find((p) => p.type === "year")?.value), month: Number(parts.find((p) => p.type === "month")?.value) };
}
export const monthId = (year: number, month: number) => `${year}-${pad(month)}`;
const shift = (year: number, month: number, by: number) => {
const index = year * 12 + (month - 1) + by;
return { year: Math.floor(index / 12), month: (index % 12) + 1 };
};
export function monthLabel(id: string): string {
const [y, m] = id.split("-").map(Number);
return new Date(Date.UTC(y, m - 1, 15)).toLocaleDateString("en-GB", { month: "long", year: "numeric", timeZone: "UTC" });
}
/** The month before the one `now` is in: what a report sent at the start of a month is about. */
export function lastFullMonth(now: Date, tz: string): string {
const cur = monthOf(now, tz);
const prev = shift(cur.year, cur.month, -1);
return monthId(prev.year, prev.month);
}
/** This month and the eleven before it, newest first. */
export function recentMonths(now: Date, tz: string, count = 12): { id: string; label: string }[] {
const cur = monthOf(now, tz);
return Array.from({ length: count }, (_, i) => {
const m = shift(cur.year, cur.month, -i);
const id = monthId(m.year, m.month);
return { id, label: monthLabel(id) };
});
}
/** The instants a month begins and ends at, on the clock of `tz`. */
export function monthBounds(id: string, tz: string): { start: Date; end: Date } {
const [y, m] = id.split("-").map(Number);
const next = shift(y, m, 1);
return { start: zonedTime(y, m, 1, 0, 0, tz), end: zonedTime(next.year, next.month, 1, 0, 0, tz) };
}
// ─── Gathering ─────────────────────────────────────────────────────────────
const round1 = (n: number) => Math.round(n * 10) / 10;
export async function buildMonthlyReport(id: string, now: Date = new Date()): Promise<MonthlyReport> {
const settings = await getSettings();
const tz = settings.notifications.timezone || "UTC";
const { start, end: fullEnd } = monthBounds(id, tz);
const partial = fullEnd.getTime() > now.getTime();
const end = partial ? now : fullEnd;
const days = Math.max(1, Math.ceil((end.getTime() - start.getTime()) / DAY_MS));
const startIso = start.toISOString();
const endIso = end.toISOString();
// Availability. Servers and integrations from the audit log's record of going away and coming back; laid over the month's days ending where the month does.
const uptime = await getUptime(end, days);
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
const integrationNames = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name]));
const rowsOf = (series: Record<number, { buckets: { k: number; d: number }[]; percent: number }>, names: Map<number, string>): AvailabilityRow[] =>
Object.entries(series)
.map(([key, s]) => ({ id: Number(key), name: names.get(Number(key)) ?? `#${key}`, percent: s.percent, downMinutes: s.buckets.reduce((n, b) => n + b.d, 0) }))
.filter((r) => names.has(r.id))
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
// Endpoints: from the daily figures, for the days the month covers. The figures are per UTC day, so a day at either end can belong to a neighbouring month
// by a few hours (more than the time zone is ahead of UTC); a month that is not over takes in today.
const [yy, mm] = id.split("-").map(Number);
const following = shift(yy, mm, 1);
const firstDay = `${id}-01`;
const lastDayExclusive = partial ? new Date(now.getTime() + DAY_MS).toISOString().slice(0, 10) : `${monthId(following.year, following.month)}-01`;
const dailyRows = await db.select().from(endpointDaily).where(and(gte(endpointDaily.day, firstDay), lt(endpointDaily.day, lastDayExclusive)));
const checkNames = new Map((await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks)).map((c) => [c.id, c.name]));
const perCheck = new Map<number, { total: number; ok: number; msSum: number; msCount: number }>();
for (const r of dailyRows) {
const t = perCheck.get(r.checkId) ?? { total: 0, ok: 0, msSum: 0, msCount: 0 };
t.total += r.total;
t.ok += r.ok;
t.msSum += r.msSum;
t.msCount += r.msCount;
perCheck.set(r.checkId, t);
}
const endpointRows: EndpointRow[] = [...perCheck]
.filter(([cid, t]) => checkNames.has(cid) && t.total > 0)
.map(([cid, t]) => ({ id: cid, name: checkNames.get(cid)!, percent: round1((t.ok / t.total) * 100), avgMs: t.msCount > 0 ? Math.round(t.msSum / t.msCount) : null, checks: t.total }))
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
// Incidents.
const all = await listIncidents({ now: end, limit: 3000 });
const inMonth = (iso: string) => iso >= startIso && iso < endIso;
const began = all.filter((i) => inMonth(i.openedAt));
const resolved = all.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
const stillOpen = all.filter((i) => i.openedAt < endIso && (i.resolvedAt === null || i.resolvedAt >= endIso)).length;
const closedHere = began.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
const minutesOf = (ms: number) => Math.max(1, Math.round(ms / 60_000));
const totalMinutes = closedHere.reduce((n, i) => n + minutesOf(i.durationMs), 0);
const byDuration = [...began].sort((a, b) => b.durationMs - a.durationMs);
const categories = new Map<string, number>();
for (const i of began) categories.set(i.category, (categories.get(i.category) ?? 0) + 1);
// Updates, by hand and by plan.
const runRows = await db.select().from(updateRuns).where(and(gte(updateRuns.startedAt, startIso), lt(updateRuns.startedAt, endIso)));
const perServer = new Map<number, { runs: number; failed: number; installed: number }>();
let installed = 0;
for (const r of runRows) {
const gained = r.status === "done" && r.updatesBefore !== null && r.updatesAfter !== null ? Math.max(0, r.updatesBefore - r.updatesAfter) : 0;
installed += gained;
const t = perServer.get(r.serverId) ?? { runs: 0, failed: 0, installed: 0 };
t.runs++;
if (r.status === "failed") t.failed++;
t.installed += gained;
perServer.set(r.serverId, t);
}
const planRows = await db.select().from(patchPlanRuns).where(and(gte(patchPlanRuns.startedAt, startIso), lt(patchPlanRuns.startedAt, endIso)));
// Restore tests, and the backups as they are now.
const tests = await db.select().from(restoreTests).where(and(gte(restoreTests.testedAt, firstDay), lt(restoreTests.testedAt, lastDayExclusive)));
const coverage = await getCachedBackupCoverage(now);
const renewalRows = await renewalsBetween(start, end);
return {
month: id,
label: monthLabel(id),
from: startIso,
to: endIso,
partial,
timezone: tz,
servers: rowsOf(uptime.servers, serverNames),
integrations: rowsOf(uptime.integrations, integrationNames),
endpoints: endpointRows,
incidents: {
began: began.length,
resolved: resolved.length,
stillOpen,
totalMinutes,
averageMinutes: closedHere.length > 0 ? Math.round(totalMinutes / closedHere.length) : null,
longest: byDuration[0] ? { id: byDuration[0].id, message: byDuration[0].message, minutes: minutesOf(byDuration[0].durationMs) } : null,
byCategory: [...categories].map(([category, count]) => ({ category, count })).sort((a, b) => b.count - a.count || a.category.localeCompare(b.category)),
list: byDuration.slice(0, MAX_LISTED).map((i) => ({ id: i.id, message: i.message, where: i.target?.name ?? i.source, severity: i.peakSeverity, openedAt: i.openedAt, resolvedAt: i.resolvedAt, minutes: minutesOf(i.durationMs) })),
},
updates: {
runs: runRows.length,
done: runRows.filter((r) => r.status === "done").length,
failed: runRows.filter((r) => r.status === "failed").length,
installed,
byServer: [...perServer]
.map(([sid, t]) => ({ name: serverNames.get(sid) ?? `#${sid}`, ...t }))
.sort((a, b) => b.runs - a.runs || a.name.localeCompare(b.name)),
planRuns: { runs: planRows.length, done: planRows.filter((r) => r.status === "done").length, failed: planRows.filter((r) => r.status === "failed").length },
},
restoreTests: tests.map((t) => ({ label: t.label, testedAt: t.testedAt, result: t.result, note: t.note })).sort((a, b) => a.testedAt.localeCompare(b.testedAt)),
backups: coverage ? { ok: coverage.counts.ok, stale: coverage.counts.stale, failed: coverage.counts.failed, none: coverage.counts.none, unknown: coverage.counts.unknown, total: coverage.counts.total, asOf: coverage.generatedAt } : null,
renewals: renewalRows.map((r) => ({ kind: r.kind, subject: r.subject, at: r.at, fromDate: r.fromDate, toDate: r.toDate })),
};
}
// ─── As a message ──────────────────────────────────────────────────────────
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
const RESULT_WORDS: Record<string, string> = { ok: "passed", partial: "partly worked", failed: "failed", not_needed: "marked as not needed" };
function average(rows: AvailabilityRow[]): number | null {
const known = rows.filter((r) => r.percent !== null);
return known.length === 0 ? null : round1(known.reduce((n, r) => n + (r.percent as number), 0) / known.length);
}
/** The report as plain text, for a notification or an email: short, a topic only where there is something to say. */
export function monthlyReportText(r: MonthlyReport): { title: string; message: string } {
const sections: string[] = [];
const lines = (heading: string, body: string[]) => body.length > 0 && sections.push(`${heading.toUpperCase()}\n${body.join("\n")}`);
const avail: string[] = [];
const serverAvg = average(r.servers);
if (serverAvg !== null) {
avail.push(`Servers: ${serverAvg}% on average (${plural(r.servers.length, "server")}).`);
for (const s of r.servers.filter((x) => x.downMinutes > 0).slice(0, 6)) avail.push(`- ${s.name}: ${s.percent}% (${spanWords(s.downMinutes)} down)`);
}
if (r.endpoints.length > 0) {
const avg = round1(r.endpoints.reduce((n, e) => n + (e.percent ?? 0), 0) / r.endpoints.length);
avail.push(`Endpoints: ${avg}% on average (${plural(r.endpoints.length, "check")}).`);
for (const e of r.endpoints.filter((x) => (x.percent ?? 100) < 100).slice(0, 6)) avail.push(`- ${e.name}: ${e.percent}%${e.avgMs !== null ? `, ${e.avgMs} ms on average` : ""}`);
}
const integrationAvg = average(r.integrations);
if (integrationAvg !== null) avail.push(`Integrations: ${integrationAvg}% on average.`);
lines("Availability", avail);
const inc = r.incidents;
if (inc.began > 0 || inc.resolved > 0 || inc.stillOpen > 0) {
const body = [`${plural(inc.began, "incident")} began, ${inc.resolved} ended, ${inc.stillOpen} still open.`];
if (inc.averageMinutes !== null) body.push(`A resolved incident took ${spanWords(inc.averageMinutes)} on average.`);
for (const i of inc.list.slice(0, 3)) body.push(`- ${i.message} — ${spanWords(i.minutes)}${i.resolvedAt ? "" : " and still open"}`);
lines("Incidents", body);
} else if (serverAvg !== null || r.endpoints.length > 0) {
lines("Incidents", ["None."]);
}
const u = r.updates;
const upd: string[] = [];
if (u.runs > 0) upd.push(`${plural(u.runs, "update run")}: ${u.done} done${u.failed > 0 ? `, ${u.failed} failed` : ""}${u.installed > 0 ? ` — ${plural(u.installed, "update")} installed` : ""}.`);
if (u.planRuns.runs > 0) upd.push(`Patch plans ran ${plural(u.planRuns.runs, "time")}${u.planRuns.failed > 0 ? `, ${u.planRuns.failed} not cleanly` : ", all cleanly"}.`);
lines("Updates", upd);
const bk: string[] = [];
for (const t of r.restoreTests) bk.push(`- Restore test of ${t.label} (${t.testedAt}) ${RESULT_WORDS[t.result] ?? t.result}`);
if (r.restoreTests.length === 0) bk.push("No restore test was recorded.");
if (r.backups) bk.push(`Backups now: ${r.backups.ok} of ${plural(r.backups.total, "guest")} recent${r.backups.failed + r.backups.stale + r.backups.none > 0 ? `; ${r.backups.failed} failed, ${r.backups.stale} stale, ${r.backups.none} without a backup` : ""}.`);
lines("Backups", bk);
lines("Renewed", r.renewals.slice(0, 10).map((x) => `- ${x.kind === "domain" ? "Domain" : "Certificate"} ${x.subject}: ${x.fromDate} → ${x.toDate}`));
const title = `Homelab monthly report — ${r.label}${r.partial ? " (so far)" : ""}`;
return { title, message: sections.length > 0 ? sections.join("\n\n") : "Nothing was recorded for this month." };
}