/** * The weekly digest: one message, sent through the notification channels, that says how the week went and what is coming — alerts now, * what was down and for how long, what happened, disks heading for full, things about to expire, updates waiting, planned maintenance. * Everything in it is already known to the app (the alerts, the uptime record, the audit log, the agents' reports); this only gathers it * into a few lines per topic and leaves out any topic with nothing to say. Plain text, so it reads in a phone notification and an email alike. */ import { db } from "../db/client.js"; import { assets, integrations, secrets, servers } from "../db/schema.js"; import { warrantyDaysLeft } from "./inventory.js"; import { env } from "../env.js"; import { getAlerts } from "./alerts.js"; import { getBackupCoverage } from "./backupCoverage.js"; import { getChangesSince } from "./changesSince.js"; import { collectDomainAlerts } from "./domainMonitor.js"; import { describeTarget, listUpcomingWindows } from "./maintenance.js"; import { notify, notifyNow } from "./notify.js"; import { parsePendingUpdates } from "./pendingUpdates.js"; import { computeSecretStatus } from "./secretStatus.js"; import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js"; import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js"; import { getSettings } from "./settingsStore.js"; import { getUptime } from "./uptimeHistory.js"; import { attentionCounts, loadAcks } from "./alertAcks.js"; import { listIncidents } from "./incidents.js"; export const DIGEST_DAYS = 7; const DAY_MS = 24 * 3600_000; const MAX_LINES = 8; export interface Digest { title: string; message: string; /** Nothing in it needs anyone: the quiet week. */ quiet: boolean; } interface Section { heading: string; lines: string[]; /** A section that only reassures ("every server stayed up") doesn't make the week un-quiet. */ attention: boolean; } /** "2 h 40 min", "35 min", "3 d 4 h". */ export function spanWords(minutes: number): string { const m = Math.max(1, Math.round(minutes)); if (m < 60) return `${m} min`; const h = Math.floor(m / 60); if (h < 24) return m % 60 && h < 10 ? `${h} h ${m % 60} min` : `${h} h`; const d = Math.floor(h / 24); return h % 24 ? `${d} d ${h % 24} h` : `${d} d`; } const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`; function listWithMore(lines: string[], max = MAX_LINES): string[] { return lines.length > max ? [...lines.slice(0, max), `…and ${lines.length - max} more`] : lines; } function whenWords(date: Date, timezone: string): string { return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", hour: "2-digit", minute: "2-digit", hourCycle: "h23" }).format(date); } function dayWords(date: Date, timezone: string): string { return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", year: "numeric" }).format(date); } export async function buildDigest(now: Date = new Date()): Promise { const settings = await getSettings(); const tz = settings.notifications.timezone || "UTC"; const sections: Section[] = []; const serverRows = await db .select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, thresholds: servers.thresholds }) .from(servers); const serverName = new Map(serverRows.map((s) => [s.id, s.name])); // ── Right now ── try { const report = await getAlerts(false); const active = report.alerts.filter((a) => !a.silenced && a.severity !== "info"); const lines: string[] = []; if (active.length === 0) lines.push(report.couldntCheck.length > 0 ? `No active alerts, but ${plural(report.couldntCheck.length, "check")} couldn't run.` : "No active alerts."); else { lines.push(`${report.counts.critical} critical, ${report.counts.warning} warning`); // Things about to expire have a section of their own, with dates; they are counted here but not listed twice. const listed = active.filter((a) => a.category !== "expiry"); lines.push(...listWithMore(listed.map((a) => `- ${a.message}`), 5)); if (listed.length < active.length) lines.push(`(${plural(active.length - listed.length, "more is", "more are")} about things expiring, below)`); } const held: string[] = []; if (report.counts.silenced > 0) held.push(`${report.counts.silenced} silenced by maintenance`); if (report.counts.grouped > 0) held.push(`${report.counts.grouped} folded under a server that is down`); if (held.length > 0) lines.push(`(${held.join(", ")})`); const acknowledged = attentionCounts(report.alerts, await loadAcks(now)).acknowledged; if (acknowledged > 0) lines.push(`(${acknowledged} of them acknowledged — somebody is on it)`); sections.push({ heading: "Right now", lines, attention: active.length > 0 || report.couldntCheck.length > 0 }); } catch (err) { console.error("[digest] couldn't read the alerts:", err); } // ── Incidents ── try { const since = now.getTime() - DIGEST_DAYS * DAY_MS; const all = await listIncidents({ now }); const begun = all.filter((i) => Date.parse(i.openedAt) >= since); const ended = all.filter((i) => i.resolvedAt !== null && Date.parse(i.resolvedAt) >= since); const open = all.filter((i) => i.open); if (begun.length > 0 || ended.length > 0) { const lines = [`${plural(begun.length, "incident")} began, ${ended.length} ended, ${open.length} still open.`]; const longest = [...ended].sort((a, b) => b.durationMs - a.durationMs).slice(0, 3); lines.push(...longest.map((i) => `- ${i.message} — lasted ${spanWords(i.durationMs / 60_000)}`)); sections.push({ heading: "Incidents", lines, attention: open.length > 0 }); } } catch (err) { console.error("[digest] couldn't read the incidents:", err); } // ── Uptime ── try { const uptime = await getUptime(now, DIGEST_DAYS); const integrationName = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name])); const rows: { name: string; down: number; percent: number }[] = []; let measured = 0; for (const [id, series] of Object.entries(uptime.servers)) { measured++; const down = series.buckets.reduce((n, b) => n + b.d, 0); if (down > 0) rows.push({ name: serverName.get(Number(id)) ?? `server ${id}`, down, percent: series.percent }); } for (const [id, series] of Object.entries(uptime.integrations)) { measured++; const down = series.buckets.reduce((n, b) => n + b.d, 0); if (down > 0) rows.push({ name: integrationName.get(Number(id)) ?? `integration ${id}`, down, percent: series.percent }); } if (measured > 0) { rows.sort((a, b) => b.down - a.down); sections.push({ heading: `Uptime, last ${DIGEST_DAYS} days`, lines: rows.length === 0 ? ["Everything stayed up."] : listWithMore(rows.map((r) => `- ${r.name}: ${r.percent}% (${spanWords(r.down)} down)`)), attention: rows.length > 0, }); } } catch (err) { console.error("[digest] couldn't read the uptime:", err); } // ── What happened ── try { const changes = await getChangesSince({ since: new Date(now.getTime() - DIGEST_DAYS * DAY_MS), now, actorId: 0, canSeeAudit: false }); if (changes.events.length > 0) { sections.push({ heading: "What happened", lines: listWithMore(changes.events.map((e) => `- ${e.title}`)), attention: changes.events.some((e) => e.tone === "bad") }); } } catch (err) { console.error("[digest] couldn't read what happened:", err); } // ── Disks heading for full ── try { const { diskUsagePercent } = settings.healthChecks; { const lines: string[] = []; for (const s of serverRows) { const own = parseThresholds(s.thresholds); const forecastDays = effectiveThresholds(settings.healthChecks, own).diskForecastDays; if (!(forecastDays > 0)) continue; for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) { lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`); } } if (lines.length > 0) sections.push({ heading: "Disks heading for full", lines: listWithMore(lines), attention: true }); } } catch (err) { console.error("[digest] couldn't read the disk forecasts:", err); } // ── About to expire ── try { const expiring: { daysLeft: number; text: string }[] = []; for (const s of await db.select().from(secrets)) { const status = computeSecretStatus(s.expiryDate, s.warnDays, now); if (status.status === "expired") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expired ${plural(Math.abs(status.daysLeft), "day")} ago` }); else if (status.status === "expiring") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expires in ${plural(status.daysLeft, "day")} (${s.expiryDate})` }); } for (const d of (await collectDomainAlerts(now)).expiring) { expiring.push({ daysLeft: d.daysLeft, text: d.daysLeft < 0 ? `Domain ${d.name} expired ${plural(Math.abs(d.daysLeft), "day")} ago` : `Domain ${d.name} expires in ${plural(d.daysLeft, "day")} (${d.expiresAt})` }); } // A warranty about to end is worth knowing while there is still time to claim on it; one that ended long ago is not news. for (const a of await db.select({ name: assets.name, warrantyUntil: assets.warrantyUntil }).from(assets)) { const days = warrantyDaysLeft(a.warrantyUntil, now); if (days !== null && days >= -30 && days <= 60) { expiring.push({ daysLeft: days, text: days < 0 ? `Warranty on ${a.name} ended ${plural(Math.abs(days), "day")} ago` : `Warranty on ${a.name} ends in ${plural(days, "day")} (${a.warrantyUntil})` }); } } if (expiring.length > 0) { expiring.sort((a, b) => a.daysLeft - b.daysLeft); sections.push({ heading: "Expiring", lines: listWithMore(expiring.map((e) => `- ${e.text}`)), attention: true }); } } catch (err) { console.error("[digest] couldn't read what is expiring:", err); } // ── Updates waiting ── try { const waiting = serverRows .map((s) => ({ name: s.name, updates: parsePendingUpdates(s.pendingUpdates), since: s.updatesPendingSince })) .filter((s) => s.updates && s.updates.count > 0); if (waiting.length > 0) { const security = waiting.reduce((n, s) => n + (s.updates!.security ?? 0), 0); const restarts = waiting.filter((s) => s.updates!.rebootRequired).length; const oldest = waiting.filter((s) => s.since).sort((a, b) => Date.parse(a.since!) - Date.parse(b.since!))[0]; const lines = [`${plural(waiting.length, "server")} ${waiting.length === 1 ? "has" : "have"} updates waiting${security > 0 ? ` (${plural(security, "security fix", "security fixes")} among them)` : ""}.`]; if (restarts > 0) lines.push(`${plural(restarts, "server")} ${restarts === 1 ? "needs" : "need"} a restart.`); if (oldest) lines.push(`Longest wait: ${oldest.name}, ${spanWords((now.getTime() - Date.parse(oldest.since!)) / 60_000)}.`); const withSecurity = waiting.filter((s) => (s.updates!.security ?? 0) > 0).sort((a, b) => (b.updates!.security ?? 0) - (a.updates!.security ?? 0)); lines.push(...listWithMore(withSecurity.map((s) => `- ${s.name}: ${s.updates!.count} waiting, ${s.updates!.security} security`), 5)); sections.push({ heading: "Updates waiting", lines, attention: security > 0 }); } } catch (err) { console.error("[digest] couldn't read the pending updates:", err); } // ── Backups ── try { const coverage = await getBackupCoverage({ now, rules: true }); const order = { failed: 0, none: 1, stale: 2 } as const; const bad = coverage.guests.filter((g): g is typeof g & { status: keyof typeof order } => g.status === "failed" || g.status === "none" || g.status === "stale").sort((a, b) => order[a.status] - order[b.status] || a.name.localeCompare(b.name)); if (bad.length > 0) { const what = (g: (typeof bad)[number]) => g.status === "failed" ? `${g.failure}` : g.status === "none" ? "no backup, and not in any backup job" : `last backup ${spanWords((now.getTime() - Date.parse(g.lastBackupAt!)) / 60_000)} ago`; const lines = [`${coverage.counts.ok} of ${plural(coverage.counts.total, "guest")} ${coverage.counts.ok === 1 ? "has" : "have"} a recent backup.`]; lines.push(...listWithMore(bad.map((g) => `- ${g.name} (${g.type === "qemu" ? "VM" : "container"} ${g.vmid}): ${what(g)}`))); sections.push({ heading: "Backups", lines, attention: true }); } } catch (err) { console.error("[digest] couldn't read the backup coverage:", err); } // ── Planned ── try { const horizon = now.getTime() + DIGEST_DAYS * DAY_MS; const groups = new Map(); for (const w of await listUpcomingWindows(now)) { if (Date.parse(w.startedAt) > horizon) continue; const target = await describeTarget(w.targetType, w.targetId); if (!target) continue; const key = `${w.startedAt}|${w.endsAt}|${w.reason ?? ""}`; const g = groups.get(key) ?? { start: w.startedAt, end: w.endsAt, reason: w.reason, names: [] }; g.names.push(target.name); groups.set(key, g); } if (groups.size > 0) { const lines = [...groups.values()] .sort((a, b) => a.start.localeCompare(b.start)) .map((g) => { const sorted = [...g.names].sort((a, b) => a.localeCompare(b)); const names = sorted.length <= 3 ? sorted.join(", ") : `${sorted.slice(0, 3).join(", ")} and ${sorted.length - 3} more`; return `- ${whenWords(new Date(g.start), tz)}, ${spanWords((Date.parse(g.end) - Date.parse(g.start)) / 60_000)}: ${names}${g.reason ? ` — ${g.reason}` : ""}`; }); sections.push({ heading: "Planned maintenance", lines: listWithMore(lines, 6), attention: false }); } } catch (err) { console.error("[digest] couldn't read the planned maintenance:", err); } const quiet = !sections.some((s) => s.attention); const envName = env.environment?.name; const title = `Homelab weekly digest${envName ? ` (${envName})` : ""}`; const body = sections.map((s) => `${s.heading.toUpperCase()}\n${s.lines.join("\n")}`).join("\n\n"); const message = `${dayWords(now, tz)}\n${quiet ? "A quiet week: nothing needs attention.\n" : ""}\n${body}`.trim(); return { title, message, quiet }; } /** Builds the digest and sends it through the channels — held with other notifications during quiet hours, unless `immediately`. */ export async function sendDigest(options: { immediately?: boolean; now?: Date } = {}): Promise { const digest = await buildDigest(options.now); if (options.immediately) await notifyNow(digest.title, digest.message); else await notify(digest.title, digest.message); return digest; }