- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics - Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents, incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs, related alerts and notes; Incidents page and a digest section - Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download, check, upload, delete, and restore staged and applied at start-up before the database opens (pre-restore copy kept); alerts and /metrics for stale or failing backups - Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
280 lines
15 KiB
TypeScript
280 lines
15 KiB
TypeScript
/**
|
|
* The weekly digest: one message, sent through the notification channels, that says how the week went and what is coming — alerts now,
|
|
* what was down and for how long, what happened, disks heading for full, things about to expire, updates waiting, planned maintenance.
|
|
* Everything in it is already known to the app (the alerts, the uptime record, the audit log, the agents' reports); this only gathers it
|
|
* into a few lines per topic and leaves out any topic with nothing to say. Plain text, so it reads in a phone notification and an email alike.
|
|
*/
|
|
import { db } from "../db/client.js";
|
|
import { assets, integrations, secrets, servers } from "../db/schema.js";
|
|
import { warrantyDaysLeft } from "./inventory.js";
|
|
import { env } from "../env.js";
|
|
import { getAlerts } from "./alerts.js";
|
|
import { getBackupCoverage } from "./backupCoverage.js";
|
|
import { getChangesSince } from "./changesSince.js";
|
|
import { collectDomainAlerts } from "./domainMonitor.js";
|
|
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
|
|
import { notify, notifyNow } from "./notify.js";
|
|
import { parsePendingUpdates } from "./pendingUpdates.js";
|
|
import { computeSecretStatus } from "./secretStatus.js";
|
|
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
|
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
|
import { getSettings } from "./settingsStore.js";
|
|
import { getUptime } from "./uptimeHistory.js";
|
|
import { attentionCounts, loadAcks } from "./alertAcks.js";
|
|
import { listIncidents } from "./incidents.js";
|
|
|
|
export const DIGEST_DAYS = 7;
|
|
const DAY_MS = 24 * 3600_000;
|
|
const MAX_LINES = 8;
|
|
|
|
export interface Digest {
|
|
title: string;
|
|
message: string;
|
|
/** Nothing in it needs anyone: the quiet week. */
|
|
quiet: boolean;
|
|
}
|
|
|
|
interface Section {
|
|
heading: string;
|
|
lines: string[];
|
|
/** A section that only reassures ("every server stayed up") doesn't make the week un-quiet. */
|
|
attention: boolean;
|
|
}
|
|
|
|
/** "2 h 40 min", "35 min", "3 d 4 h". */
|
|
export function spanWords(minutes: number): string {
|
|
const m = Math.max(1, Math.round(minutes));
|
|
if (m < 60) return `${m} min`;
|
|
const h = Math.floor(m / 60);
|
|
if (h < 24) return m % 60 && h < 10 ? `${h} h ${m % 60} min` : `${h} h`;
|
|
const d = Math.floor(h / 24);
|
|
return h % 24 ? `${d} d ${h % 24} h` : `${d} d`;
|
|
}
|
|
|
|
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
|
|
|
function listWithMore(lines: string[], max = MAX_LINES): string[] {
|
|
return lines.length > max ? [...lines.slice(0, max), `…and ${lines.length - max} more`] : lines;
|
|
}
|
|
|
|
function whenWords(date: Date, timezone: string): string {
|
|
return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", hour: "2-digit", minute: "2-digit", hourCycle: "h23" }).format(date);
|
|
}
|
|
|
|
function dayWords(date: Date, timezone: string): string {
|
|
return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", year: "numeric" }).format(date);
|
|
}
|
|
|
|
export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
|
const settings = await getSettings();
|
|
const tz = settings.notifications.timezone || "UTC";
|
|
const sections: Section[] = [];
|
|
|
|
const serverRows = await db
|
|
.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, thresholds: servers.thresholds })
|
|
.from(servers);
|
|
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
|
|
|
// ── Right now ──
|
|
try {
|
|
const report = await getAlerts(false);
|
|
const active = report.alerts.filter((a) => !a.silenced && a.severity !== "info");
|
|
const lines: string[] = [];
|
|
if (active.length === 0) lines.push(report.couldntCheck.length > 0 ? `No active alerts, but ${plural(report.couldntCheck.length, "check")} couldn't run.` : "No active alerts.");
|
|
else {
|
|
lines.push(`${report.counts.critical} critical, ${report.counts.warning} warning`);
|
|
// Things about to expire have a section of their own, with dates; they are counted here but not listed twice.
|
|
const listed = active.filter((a) => a.category !== "expiry");
|
|
lines.push(...listWithMore(listed.map((a) => `- ${a.message}`), 5));
|
|
if (listed.length < active.length) lines.push(`(${plural(active.length - listed.length, "more is", "more are")} about things expiring, below)`);
|
|
}
|
|
const held: string[] = [];
|
|
if (report.counts.silenced > 0) held.push(`${report.counts.silenced} silenced by maintenance`);
|
|
if (report.counts.grouped > 0) held.push(`${report.counts.grouped} folded under a server that is down`);
|
|
if (held.length > 0) lines.push(`(${held.join(", ")})`);
|
|
const acknowledged = attentionCounts(report.alerts, await loadAcks(now)).acknowledged;
|
|
if (acknowledged > 0) lines.push(`(${acknowledged} of them acknowledged — somebody is on it)`);
|
|
sections.push({ heading: "Right now", lines, attention: active.length > 0 || report.couldntCheck.length > 0 });
|
|
} catch (err) {
|
|
console.error("[digest] couldn't read the alerts:", err);
|
|
}
|
|
|
|
// ── Incidents ──
|
|
try {
|
|
const since = now.getTime() - DIGEST_DAYS * DAY_MS;
|
|
const all = await listIncidents({ now });
|
|
const begun = all.filter((i) => Date.parse(i.openedAt) >= since);
|
|
const ended = all.filter((i) => i.resolvedAt !== null && Date.parse(i.resolvedAt) >= since);
|
|
const open = all.filter((i) => i.open);
|
|
if (begun.length > 0 || ended.length > 0) {
|
|
const lines = [`${plural(begun.length, "incident")} began, ${ended.length} ended, ${open.length} still open.`];
|
|
const longest = [...ended].sort((a, b) => b.durationMs - a.durationMs).slice(0, 3);
|
|
lines.push(...longest.map((i) => `- ${i.message} — lasted ${spanWords(i.durationMs / 60_000)}`));
|
|
sections.push({ heading: "Incidents", lines, attention: open.length > 0 });
|
|
}
|
|
} catch (err) {
|
|
console.error("[digest] couldn't read the incidents:", err);
|
|
}
|
|
|
|
// ── Uptime ──
|
|
try {
|
|
const uptime = await getUptime(now, DIGEST_DAYS);
|
|
const integrationName = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name]));
|
|
const rows: { name: string; down: number; percent: number }[] = [];
|
|
let measured = 0;
|
|
for (const [id, series] of Object.entries(uptime.servers)) {
|
|
measured++;
|
|
const down = series.buckets.reduce((n, b) => n + b.d, 0);
|
|
if (down > 0) rows.push({ name: serverName.get(Number(id)) ?? `server ${id}`, down, percent: series.percent });
|
|
}
|
|
for (const [id, series] of Object.entries(uptime.integrations)) {
|
|
measured++;
|
|
const down = series.buckets.reduce((n, b) => n + b.d, 0);
|
|
if (down > 0) rows.push({ name: integrationName.get(Number(id)) ?? `integration ${id}`, down, percent: series.percent });
|
|
}
|
|
if (measured > 0) {
|
|
rows.sort((a, b) => b.down - a.down);
|
|
sections.push({
|
|
heading: `Uptime, last ${DIGEST_DAYS} days`,
|
|
lines: rows.length === 0 ? ["Everything stayed up."] : listWithMore(rows.map((r) => `- ${r.name}: ${r.percent}% (${spanWords(r.down)} down)`)),
|
|
attention: rows.length > 0,
|
|
});
|
|
}
|
|
} catch (err) {
|
|
console.error("[digest] couldn't read the uptime:", err);
|
|
}
|
|
|
|
// ── What happened ──
|
|
try {
|
|
const changes = await getChangesSince({ since: new Date(now.getTime() - DIGEST_DAYS * DAY_MS), now, actorId: 0, canSeeAudit: false });
|
|
if (changes.events.length > 0) {
|
|
sections.push({ heading: "What happened", lines: listWithMore(changes.events.map((e) => `- ${e.title}`)), attention: changes.events.some((e) => e.tone === "bad") });
|
|
}
|
|
} catch (err) {
|
|
console.error("[digest] couldn't read what happened:", err);
|
|
}
|
|
|
|
// ── Disks heading for full ──
|
|
try {
|
|
const { diskUsagePercent } = settings.healthChecks;
|
|
{
|
|
const lines: string[] = [];
|
|
for (const s of serverRows) {
|
|
const own = parseThresholds(s.thresholds);
|
|
const forecastDays = effectiveThresholds(settings.healthChecks, own).diskForecastDays;
|
|
if (!(forecastDays > 0)) continue;
|
|
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
|
|
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
|
|
}
|
|
}
|
|
if (lines.length > 0) sections.push({ heading: "Disks heading for full", lines: listWithMore(lines), attention: true });
|
|
}
|
|
} catch (err) {
|
|
console.error("[digest] couldn't read the disk forecasts:", err);
|
|
}
|
|
|
|
// ── About to expire ──
|
|
try {
|
|
const expiring: { daysLeft: number; text: string }[] = [];
|
|
for (const s of await db.select().from(secrets)) {
|
|
const status = computeSecretStatus(s.expiryDate, s.warnDays, now);
|
|
if (status.status === "expired") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expired ${plural(Math.abs(status.daysLeft), "day")} ago` });
|
|
else if (status.status === "expiring") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expires in ${plural(status.daysLeft, "day")} (${s.expiryDate})` });
|
|
}
|
|
for (const d of (await collectDomainAlerts(now)).expiring) {
|
|
expiring.push({ daysLeft: d.daysLeft, text: d.daysLeft < 0 ? `Domain ${d.name} expired ${plural(Math.abs(d.daysLeft), "day")} ago` : `Domain ${d.name} expires in ${plural(d.daysLeft, "day")} (${d.expiresAt})` });
|
|
}
|
|
// A warranty about to end is worth knowing while there is still time to claim on it; one that ended long ago is not news.
|
|
for (const a of await db.select({ name: assets.name, warrantyUntil: assets.warrantyUntil }).from(assets)) {
|
|
const days = warrantyDaysLeft(a.warrantyUntil, now);
|
|
if (days !== null && days >= -30 && days <= 60) {
|
|
expiring.push({ daysLeft: days, text: days < 0 ? `Warranty on ${a.name} ended ${plural(Math.abs(days), "day")} ago` : `Warranty on ${a.name} ends in ${plural(days, "day")} (${a.warrantyUntil})` });
|
|
}
|
|
}
|
|
if (expiring.length > 0) {
|
|
expiring.sort((a, b) => a.daysLeft - b.daysLeft);
|
|
sections.push({ heading: "Expiring", lines: listWithMore(expiring.map((e) => `- ${e.text}`)), attention: true });
|
|
}
|
|
} catch (err) {
|
|
console.error("[digest] couldn't read what is expiring:", err);
|
|
}
|
|
|
|
// ── Updates waiting ──
|
|
try {
|
|
const waiting = serverRows
|
|
.map((s) => ({ name: s.name, updates: parsePendingUpdates(s.pendingUpdates), since: s.updatesPendingSince }))
|
|
.filter((s) => s.updates && s.updates.count > 0);
|
|
if (waiting.length > 0) {
|
|
const security = waiting.reduce((n, s) => n + (s.updates!.security ?? 0), 0);
|
|
const restarts = waiting.filter((s) => s.updates!.rebootRequired).length;
|
|
const oldest = waiting.filter((s) => s.since).sort((a, b) => Date.parse(a.since!) - Date.parse(b.since!))[0];
|
|
const lines = [`${plural(waiting.length, "server")} ${waiting.length === 1 ? "has" : "have"} updates waiting${security > 0 ? ` (${plural(security, "security fix", "security fixes")} among them)` : ""}.`];
|
|
if (restarts > 0) lines.push(`${plural(restarts, "server")} ${restarts === 1 ? "needs" : "need"} a restart.`);
|
|
if (oldest) lines.push(`Longest wait: ${oldest.name}, ${spanWords((now.getTime() - Date.parse(oldest.since!)) / 60_000)}.`);
|
|
const withSecurity = waiting.filter((s) => (s.updates!.security ?? 0) > 0).sort((a, b) => (b.updates!.security ?? 0) - (a.updates!.security ?? 0));
|
|
lines.push(...listWithMore(withSecurity.map((s) => `- ${s.name}: ${s.updates!.count} waiting, ${s.updates!.security} security`), 5));
|
|
sections.push({ heading: "Updates waiting", lines, attention: security > 0 });
|
|
}
|
|
} catch (err) {
|
|
console.error("[digest] couldn't read the pending updates:", err);
|
|
}
|
|
|
|
// ── Backups ──
|
|
try {
|
|
const coverage = await getBackupCoverage({ now, rules: true });
|
|
const order = { failed: 0, none: 1, stale: 2 } as const;
|
|
const bad = coverage.guests.filter((g): g is typeof g & { status: keyof typeof order } => g.status === "failed" || g.status === "none" || g.status === "stale").sort((a, b) => order[a.status] - order[b.status] || a.name.localeCompare(b.name));
|
|
if (bad.length > 0) {
|
|
const what = (g: (typeof bad)[number]) =>
|
|
g.status === "failed" ? `${g.failure}` : g.status === "none" ? "no backup, and not in any backup job" : `last backup ${spanWords((now.getTime() - Date.parse(g.lastBackupAt!)) / 60_000)} ago`;
|
|
const lines = [`${coverage.counts.ok} of ${plural(coverage.counts.total, "guest")} ${coverage.counts.ok === 1 ? "has" : "have"} a recent backup.`];
|
|
lines.push(...listWithMore(bad.map((g) => `- ${g.name} (${g.type === "qemu" ? "VM" : "container"} ${g.vmid}): ${what(g)}`)));
|
|
sections.push({ heading: "Backups", lines, attention: true });
|
|
}
|
|
} catch (err) {
|
|
console.error("[digest] couldn't read the backup coverage:", err);
|
|
}
|
|
|
|
// ── Planned ──
|
|
try {
|
|
const horizon = now.getTime() + DIGEST_DAYS * DAY_MS;
|
|
const groups = new Map<string, { start: string; end: string; reason: string | null; names: string[] }>();
|
|
for (const w of await listUpcomingWindows(now)) {
|
|
if (Date.parse(w.startedAt) > horizon) continue;
|
|
const target = await describeTarget(w.targetType, w.targetId);
|
|
if (!target) continue;
|
|
const key = `${w.startedAt}|${w.endsAt}|${w.reason ?? ""}`;
|
|
const g = groups.get(key) ?? { start: w.startedAt, end: w.endsAt, reason: w.reason, names: [] };
|
|
g.names.push(target.name);
|
|
groups.set(key, g);
|
|
}
|
|
if (groups.size > 0) {
|
|
const lines = [...groups.values()]
|
|
.sort((a, b) => a.start.localeCompare(b.start))
|
|
.map((g) => {
|
|
const sorted = [...g.names].sort((a, b) => a.localeCompare(b));
|
|
const names = sorted.length <= 3 ? sorted.join(", ") : `${sorted.slice(0, 3).join(", ")} and ${sorted.length - 3} more`;
|
|
return `- ${whenWords(new Date(g.start), tz)}, ${spanWords((Date.parse(g.end) - Date.parse(g.start)) / 60_000)}: ${names}${g.reason ? ` — ${g.reason}` : ""}`;
|
|
});
|
|
sections.push({ heading: "Planned maintenance", lines: listWithMore(lines, 6), attention: false });
|
|
}
|
|
} catch (err) {
|
|
console.error("[digest] couldn't read the planned maintenance:", err);
|
|
}
|
|
|
|
const quiet = !sections.some((s) => s.attention);
|
|
const envName = env.environment?.name;
|
|
const title = `Homelab weekly digest${envName ? ` (${envName})` : ""}`;
|
|
const body = sections.map((s) => `${s.heading.toUpperCase()}\n${s.lines.join("\n")}`).join("\n\n");
|
|
const message = `${dayWords(now, tz)}\n${quiet ? "A quiet week: nothing needs attention.\n" : ""}\n${body}`.trim();
|
|
return { title, message, quiet };
|
|
}
|
|
|
|
/** Builds the digest and sends it through the channels — held with other notifications during quiet hours, unless `immediately`. */
|
|
export async function sendDigest(options: { immediately?: boolean; now?: Date } = {}): Promise<Digest> {
|
|
const digest = await buildDigest(options.now);
|
|
if (options.immediately) await notifyNow(digest.title, digest.message);
|
|
else await notify(digest.title, digest.message);
|
|
return digest;
|
|
}
|