Files
Homelab-manager/server/src/services/weeklyDigest.ts
T
bobbanandClaude Sonnet 5.5 68b41b5586 A year of history, one chart, and a read API with personal tokens
History: server load, memory and disks are also kept as hourly (four months) and
daily (over two years) averages with lowest and highest; endpoint answers as
hourly (four months) beside the daily figures. Charts ask for 24 hours to a year
and are drawn from whichever is fine enough. A disk that two weeks call flat is
looked at again over ninety days for the forecast. Built once from what is kept.

One chart: TimeChart replaces the sparkline and the multi-line chart: hover and
arrow-key readout, a lowest/highest band, gaps shown as gaps, the same ranges
everywhere and the choice remembered. Used for server trends, comparing servers,
a check's availability and answer times (Endpoints > History) and a server's
availability.

Read API: /api/v1 (servers, metrics, alerts, incidents, inventory, network,
endpoints and their history, maintenance) with personal read-only tokens made
under the user menu: shown once, hashed, 30/90/365 days or none, ten at most,
removed with their owner. GET only. Documented in API.md.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-20 20:22:58 +02:00

285 lines
16 KiB
TypeScript

/**
* The weekly digest: one message, sent through the notification channels, that says how the week went and what is coming — alerts now,
* what was down and for how long, what happened, disks heading for full, things about to expire, updates waiting, planned maintenance.
* Everything in it is already known to the app (the alerts, the uptime record, the audit log, the agents' reports); this only gathers it
* into a few lines per topic and leaves out any topic with nothing to say. Plain text, so it reads in a phone notification and an email alike.
*/
import { db } from "../db/client.js";
import { assets, integrations, secrets, servers } from "../db/schema.js";
import { warrantyDaysLeft } from "./inventory.js";
import { env } from "../env.js";
import { getAlerts } from "./alerts.js";
import { getBackupCoverage } from "./backupCoverage.js";
import { getChangesSince } from "./changesSince.js";
import { collectDomainAlerts } from "./domainMonitor.js";
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
import { notify, notifyNow } from "./notify.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { computeSecretStatus } from "./secretStatus.js";
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
import { attentionCounts, loadAcks } from "./alertAcks.js";
import { listIncidents } from "./incidents.js";
export const DIGEST_DAYS = 7;
const DAY_MS = 24 * 3600_000;
const MAX_LINES = 8;
export interface Digest {
title: string;
message: string;
/** Nothing in it needs anyone: the quiet week. */
quiet: boolean;
}
interface Section {
heading: string;
lines: string[];
/** A section that only reassures ("every server stayed up") doesn't make the week un-quiet. */
attention: boolean;
}
/** "2 h 40 min", "35 min", "3 d 4 h". */
export function spanWords(minutes: number): string {
const m = Math.max(1, Math.round(minutes));
if (m < 60) return `${m} min`;
const h = Math.floor(m / 60);
if (h < 24) return m % 60 && h < 10 ? `${h} h ${m % 60} min` : `${h} h`;
const d = Math.floor(h / 24);
return h % 24 ? `${d} d ${h % 24} h` : `${d} d`;
}
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
function listWithMore(lines: string[], max = MAX_LINES): string[] {
return lines.length > max ? [...lines.slice(0, max), `…and ${lines.length - max} more`] : lines;
}
function whenWords(date: Date, timezone: string): string {
return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", hour: "2-digit", minute: "2-digit", hourCycle: "h23" }).format(date);
}
function dayWords(date: Date, timezone: string): string {
return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", year: "numeric" }).format(date);
}
export async function buildDigest(now: Date = new Date()): Promise<Digest> {
const settings = await getSettings();
const tz = settings.notifications.timezone || "UTC";
const sections: Section[] = [];
const serverRows = await db
.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, thresholds: servers.thresholds })
.from(servers);
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
// ── Right now ──
try {
const report = await getAlerts(false);
const active = report.alerts.filter((a) => !a.silenced && a.severity !== "info");
const lines: string[] = [];
if (active.length === 0) lines.push(report.couldntCheck.length > 0 ? `No active alerts, but ${plural(report.couldntCheck.length, "check")} couldn't run.` : "No active alerts.");
else {
lines.push(`${report.counts.critical} critical, ${report.counts.warning} warning`);
// Things about to expire have a section of their own, with dates; they are counted here but not listed twice.
const listed = active.filter((a) => a.category !== "expiry");
lines.push(...listWithMore(listed.map((a) => `- ${a.message}`), 5));
if (listed.length < active.length) lines.push(`(${plural(active.length - listed.length, "more is", "more are")} about things expiring, below)`);
}
const held: string[] = [];
if (report.counts.silenced > 0) held.push(`${report.counts.silenced} silenced by maintenance`);
if (report.counts.grouped > 0) held.push(`${report.counts.grouped} folded under a server that is down`);
if (held.length > 0) lines.push(`(${held.join(", ")})`);
const acknowledged = attentionCounts(report.alerts, await loadAcks(now)).acknowledged;
if (acknowledged > 0) lines.push(`(${acknowledged} of them acknowledged — somebody is on it)`);
sections.push({ heading: "Right now", lines, attention: active.length > 0 || report.couldntCheck.length > 0 });
} catch (err) {
console.error("[digest] couldn't read the alerts:", err);
}
// ── Incidents ──
try {
const since = now.getTime() - DIGEST_DAYS * DAY_MS;
const all = await listIncidents({ now });
const begun = all.filter((i) => Date.parse(i.openedAt) >= since);
const ended = all.filter((i) => i.resolvedAt !== null && Date.parse(i.resolvedAt) >= since);
const open = all.filter((i) => i.open);
if (begun.length > 0 || ended.length > 0) {
const lines = [`${plural(begun.length, "incident")} began, ${ended.length} ended, ${open.length} still open.`];
const longest = [...ended].sort((a, b) => b.durationMs - a.durationMs).slice(0, 3);
lines.push(...longest.map((i) => `- ${i.message} — lasted ${spanWords(i.durationMs / 60_000)}`));
sections.push({ heading: "Incidents", lines, attention: open.length > 0 });
}
} catch (err) {
console.error("[digest] couldn't read the incidents:", err);
}
// ── Uptime ──
try {
const uptime = await getUptime(now, DIGEST_DAYS);
const integrationName = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name]));
const rows: { name: string; down: number; percent: number }[] = [];
let measured = 0;
for (const [id, series] of Object.entries(uptime.servers)) {
measured++;
const down = series.buckets.reduce((n, b) => n + b.d, 0);
if (down > 0) rows.push({ name: serverName.get(Number(id)) ?? `server ${id}`, down, percent: series.percent });
}
for (const [id, series] of Object.entries(uptime.integrations)) {
measured++;
const down = series.buckets.reduce((n, b) => n + b.d, 0);
if (down > 0) rows.push({ name: integrationName.get(Number(id)) ?? `integration ${id}`, down, percent: series.percent });
}
if (measured > 0) {
rows.sort((a, b) => b.down - a.down);
sections.push({
heading: `Uptime, last ${DIGEST_DAYS} days`,
lines: rows.length === 0 ? ["Everything stayed up."] : listWithMore(rows.map((r) => `- ${r.name}: ${r.percent}% (${spanWords(r.down)} down)`)),
attention: rows.length > 0,
});
}
} catch (err) {
console.error("[digest] couldn't read the uptime:", err);
}
// ── What happened ──
try {
const changes = await getChangesSince({ since: new Date(now.getTime() - DIGEST_DAYS * DAY_MS), now, actorId: 0, canSeeAudit: false });
if (changes.events.length > 0) {
sections.push({ heading: "What happened", lines: listWithMore(changes.events.map((e) => `- ${e.title}`)), attention: changes.events.some((e) => e.tone === "bad") });
}
} catch (err) {
console.error("[digest] couldn't read what happened:", err);
}
// ── Disks heading for full ──
try {
const { diskUsagePercent } = settings.healthChecks;
{
const lines: string[] = [];
for (const s of serverRows) {
const own = parseThresholds(s.thresholds);
const forecastDays = effectiveThresholds(settings.healthChecks, own).diskForecastDays;
if (!(forecastDays > 0)) continue;
for (const f of disksFillingSoon(await getForecastSeries(s.id, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
}
}
if (lines.length > 0) sections.push({ heading: "Disks heading for full", lines: listWithMore(lines), attention: true });
}
} catch (err) {
console.error("[digest] couldn't read the disk forecasts:", err);
}
// ── About to expire ──
try {
const expiring: { daysLeft: number; text: string }[] = [];
for (const s of await db.select().from(secrets)) {
const status = computeSecretStatus(s.expiryDate, s.warnDays, now);
if (status.status === "expired") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expired ${plural(Math.abs(status.daysLeft), "day")} ago` });
else if (status.status === "expiring") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expires in ${plural(status.daysLeft, "day")} (${s.expiryDate})` });
}
for (const d of (await collectDomainAlerts(now)).expiring) {
expiring.push({ daysLeft: d.daysLeft, text: d.daysLeft < 0 ? `Domain ${d.name} expired ${plural(Math.abs(d.daysLeft), "day")} ago` : `Domain ${d.name} expires in ${plural(d.daysLeft, "day")} (${d.expiresAt})` });
}
// A warranty about to end is worth knowing while there is still time to claim on it; one that ended long ago is not news.
for (const a of await db.select({ name: assets.name, warrantyUntil: assets.warrantyUntil, endOfSupport: assets.endOfSupport }).from(assets)) {
const days = warrantyDaysLeft(a.warrantyUntil, now);
if (days !== null && days >= -30 && days <= 60) {
expiring.push({ daysLeft: days, text: days < 0 ? `Warranty on ${a.name} ended ${plural(Math.abs(days), "day")} ago` : `Warranty on ${a.name} ends in ${plural(days, "day")} (${a.warrantyUntil})` });
}
// The end of support gives more notice: it is when something has to be replaced, not claimed on.
const supportDays = warrantyDaysLeft(a.endOfSupport, now);
if (supportDays !== null && supportDays >= -30 && supportDays <= 90) {
expiring.push({ daysLeft: supportDays, text: supportDays < 0 ? `Support for ${a.name} ended ${plural(Math.abs(supportDays), "day")} ago` : `Support for ${a.name} ends in ${plural(supportDays, "day")} (${a.endOfSupport})` });
}
}
if (expiring.length > 0) {
expiring.sort((a, b) => a.daysLeft - b.daysLeft);
sections.push({ heading: "Expiring", lines: listWithMore(expiring.map((e) => `- ${e.text}`)), attention: true });
}
} catch (err) {
console.error("[digest] couldn't read what is expiring:", err);
}
// ── Updates waiting ──
try {
const waiting = serverRows
.map((s) => ({ name: s.name, updates: parsePendingUpdates(s.pendingUpdates), since: s.updatesPendingSince }))
.filter((s) => s.updates && s.updates.count > 0);
if (waiting.length > 0) {
const security = waiting.reduce((n, s) => n + (s.updates!.security ?? 0), 0);
const restarts = waiting.filter((s) => s.updates!.rebootRequired).length;
const oldest = waiting.filter((s) => s.since).sort((a, b) => Date.parse(a.since!) - Date.parse(b.since!))[0];
const lines = [`${plural(waiting.length, "server")} ${waiting.length === 1 ? "has" : "have"} updates waiting${security > 0 ? ` (${plural(security, "security fix", "security fixes")} among them)` : ""}.`];
if (restarts > 0) lines.push(`${plural(restarts, "server")} ${restarts === 1 ? "needs" : "need"} a restart.`);
if (oldest) lines.push(`Longest wait: ${oldest.name}, ${spanWords((now.getTime() - Date.parse(oldest.since!)) / 60_000)}.`);
const withSecurity = waiting.filter((s) => (s.updates!.security ?? 0) > 0).sort((a, b) => (b.updates!.security ?? 0) - (a.updates!.security ?? 0));
lines.push(...listWithMore(withSecurity.map((s) => `- ${s.name}: ${s.updates!.count} waiting, ${s.updates!.security} security`), 5));
sections.push({ heading: "Updates waiting", lines, attention: security > 0 });
}
} catch (err) {
console.error("[digest] couldn't read the pending updates:", err);
}
// ── Backups ──
try {
const coverage = await getBackupCoverage({ now, rules: true });
const order = { failed: 0, none: 1, stale: 2 } as const;
const bad = coverage.guests.filter((g): g is typeof g & { status: keyof typeof order } => g.status === "failed" || g.status === "none" || g.status === "stale").sort((a, b) => order[a.status] - order[b.status] || a.name.localeCompare(b.name));
if (bad.length > 0) {
const what = (g: (typeof bad)[number]) =>
g.status === "failed" ? `${g.failure}` : g.status === "none" ? "no backup, and not in any backup job" : `last backup ${spanWords((now.getTime() - Date.parse(g.lastBackupAt!)) / 60_000)} ago`;
const lines = [`${coverage.counts.ok} of ${plural(coverage.counts.total, "guest")} ${coverage.counts.ok === 1 ? "has" : "have"} a recent backup.`];
lines.push(...listWithMore(bad.map((g) => `- ${g.name} (${g.type === "qemu" ? "VM" : "container"} ${g.vmid}): ${what(g)}`)));
sections.push({ heading: "Backups", lines, attention: true });
}
} catch (err) {
console.error("[digest] couldn't read the backup coverage:", err);
}
// ── Planned ──
try {
const horizon = now.getTime() + DIGEST_DAYS * DAY_MS;
const groups = new Map<string, { start: string; end: string; reason: string | null; names: string[] }>();
for (const w of await listUpcomingWindows(now)) {
if (Date.parse(w.startedAt) > horizon) continue;
const target = await describeTarget(w.targetType, w.targetId);
if (!target) continue;
const key = `${w.startedAt}|${w.endsAt}|${w.reason ?? ""}`;
const g = groups.get(key) ?? { start: w.startedAt, end: w.endsAt, reason: w.reason, names: [] };
g.names.push(target.name);
groups.set(key, g);
}
if (groups.size > 0) {
const lines = [...groups.values()]
.sort((a, b) => a.start.localeCompare(b.start))
.map((g) => {
const sorted = [...g.names].sort((a, b) => a.localeCompare(b));
const names = sorted.length <= 3 ? sorted.join(", ") : `${sorted.slice(0, 3).join(", ")} and ${sorted.length - 3} more`;
return `- ${whenWords(new Date(g.start), tz)}, ${spanWords((Date.parse(g.end) - Date.parse(g.start)) / 60_000)}: ${names}${g.reason ? ` — ${g.reason}` : ""}`;
});
sections.push({ heading: "Planned maintenance", lines: listWithMore(lines, 6), attention: false });
}
} catch (err) {
console.error("[digest] couldn't read the planned maintenance:", err);
}
const quiet = !sections.some((s) => s.attention);
const envName = env.environment?.name;
const title = `Homelab weekly digest${envName ? ` (${envName})` : ""}`;
const body = sections.map((s) => `${s.heading.toUpperCase()}\n${s.lines.join("\n")}`).join("\n\n");
const message = `${dayWords(now, tz)}\n${quiet ? "A quiet week: nothing needs attention.\n" : ""}\n${body}`.trim();
return { title, message, quiet };
}
/** Builds the digest and sends it through the channels — held with other notifications during quiet hours, unless `immediately`. */
export async function sendDigest(options: { immediately?: boolean; now?: Date } = {}): Promise<Digest> {
const digest = await buildDigest(options.now);
if (options.immediately) await notifyNow(digest.title, digest.message);
else await notify(digest.title, digest.message);
return digest;
}