Add a weekly digest and report which version of the agent each server runs
- Weekly digest through the notification channels: alerts now, a week of uptime, what happened, disks heading for full, expiring secrets and domains, updates waiting, planned maintenance. Off until switched on in Settings -> Notifications; Preview and Send it now there. Setting key 'digest'. - Agents say their version in every report (Linux and Windows scripts are version 2); the server compares it with the scripts it serves and marks agents that are behind on the Servers, server and Updates pages. servers.agent_version, migration 0026. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3da5c639b1
commit
aa94d84f6c
28 files changed
+3179
-19
No files matched your search
@@ -0,0 +1,229 @@
|
||||
/**
|
||||
* The weekly digest: one message, sent through the notification channels, that says how the week went and what is coming — alerts now,
|
||||
* what was down and for how long, what happened, disks heading for full, things about to expire, updates waiting, planned maintenance.
|
||||
* Everything in it is already known to the app (the alerts, the uptime record, the audit log, the agents' reports); this only gathers it
|
||||
* into a few lines per topic and leaves out any topic with nothing to say. Plain text, so it reads in a phone notification and an email alike.
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, secrets, servers } from "../db/schema.js";
|
||||
import { env } from "../env.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { getChangesSince } from "./changesSince.js";
|
||||
import { collectDomainAlerts } from "./domainMonitor.js";
|
||||
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
|
||||
import { notify, notifyNow } from "./notify.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
|
||||
export const DIGEST_DAYS = 7;
|
||||
const DAY_MS = 24 * 3600_000;
|
||||
const MAX_LINES = 8;
|
||||
|
||||
export interface Digest {
|
||||
title: string;
|
||||
message: string;
|
||||
/** Nothing in it needs anyone: the quiet week. */
|
||||
quiet: boolean;
|
||||
}
|
||||
|
||||
interface Section {
|
||||
heading: string;
|
||||
lines: string[];
|
||||
/** A section that only reassures ("every server stayed up") doesn't make the week un-quiet. */
|
||||
attention: boolean;
|
||||
}
|
||||
|
||||
/** "2 h 40 min", "35 min", "3 d 4 h". */
|
||||
export function spanWords(minutes: number): string {
|
||||
const m = Math.max(1, Math.round(minutes));
|
||||
if (m < 60) return `${m} min`;
|
||||
const h = Math.floor(m / 60);
|
||||
if (h < 24) return m % 60 && h < 10 ? `${h} h ${m % 60} min` : `${h} h`;
|
||||
const d = Math.floor(h / 24);
|
||||
return h % 24 ? `${d} d ${h % 24} h` : `${d} d`;
|
||||
}
|
||||
|
||||
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
||||
|
||||
function listWithMore(lines: string[], max = MAX_LINES): string[] {
|
||||
return lines.length > max ? [...lines.slice(0, max), `…and ${lines.length - max} more`] : lines;
|
||||
}
|
||||
|
||||
function whenWords(date: Date, timezone: string): string {
|
||||
return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", hour: "2-digit", minute: "2-digit", hourCycle: "h23" }).format(date);
|
||||
}
|
||||
|
||||
function dayWords(date: Date, timezone: string): string {
|
||||
return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", year: "numeric" }).format(date);
|
||||
}
|
||||
|
||||
export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
const settings = await getSettings();
|
||||
const tz = settings.notifications.timezone || "UTC";
|
||||
const sections: Section[] = [];
|
||||
|
||||
const serverRows = await db
|
||||
.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince })
|
||||
.from(servers);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
|
||||
// ── Right now ──
|
||||
try {
|
||||
const report = await getAlerts(false);
|
||||
const active = report.alerts.filter((a) => !a.silenced && a.severity !== "info");
|
||||
const lines: string[] = [];
|
||||
if (active.length === 0) lines.push(report.couldntCheck.length > 0 ? `No active alerts, but ${plural(report.couldntCheck.length, "check")} couldn't run.` : "No active alerts.");
|
||||
else {
|
||||
lines.push(`${report.counts.critical} critical, ${report.counts.warning} warning`);
|
||||
// Things about to expire have a section of their own, with dates; they are counted here but not listed twice.
|
||||
const listed = active.filter((a) => a.category !== "expiry");
|
||||
lines.push(...listWithMore(listed.map((a) => `- ${a.message}`), 5));
|
||||
if (listed.length < active.length) lines.push(`(${plural(active.length - listed.length, "more is", "more are")} about things expiring, below)`);
|
||||
}
|
||||
const held: string[] = [];
|
||||
if (report.counts.silenced > 0) held.push(`${report.counts.silenced} silenced by maintenance`);
|
||||
if (report.counts.grouped > 0) held.push(`${report.counts.grouped} folded under a server that is down`);
|
||||
if (held.length > 0) lines.push(`(${held.join(", ")})`);
|
||||
sections.push({ heading: "Right now", lines, attention: active.length > 0 || report.couldntCheck.length > 0 });
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the alerts:", err);
|
||||
}
|
||||
|
||||
// ── Uptime ──
|
||||
try {
|
||||
const uptime = await getUptime(now, DIGEST_DAYS);
|
||||
const integrationName = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name]));
|
||||
const rows: { name: string; down: number; percent: number }[] = [];
|
||||
let measured = 0;
|
||||
for (const [id, series] of Object.entries(uptime.servers)) {
|
||||
measured++;
|
||||
const down = series.buckets.reduce((n, b) => n + b.d, 0);
|
||||
if (down > 0) rows.push({ name: serverName.get(Number(id)) ?? `server ${id}`, down, percent: series.percent });
|
||||
}
|
||||
for (const [id, series] of Object.entries(uptime.integrations)) {
|
||||
measured++;
|
||||
const down = series.buckets.reduce((n, b) => n + b.d, 0);
|
||||
if (down > 0) rows.push({ name: integrationName.get(Number(id)) ?? `integration ${id}`, down, percent: series.percent });
|
||||
}
|
||||
if (measured > 0) {
|
||||
rows.sort((a, b) => b.down - a.down);
|
||||
sections.push({
|
||||
heading: `Uptime, last ${DIGEST_DAYS} days`,
|
||||
lines: rows.length === 0 ? ["Everything stayed up."] : listWithMore(rows.map((r) => `- ${r.name}: ${r.percent}% (${spanWords(r.down)} down)`)),
|
||||
attention: rows.length > 0,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the uptime:", err);
|
||||
}
|
||||
|
||||
// ── What happened ──
|
||||
try {
|
||||
const changes = await getChangesSince({ since: new Date(now.getTime() - DIGEST_DAYS * DAY_MS), now, actorId: 0, canSeeAudit: false });
|
||||
if (changes.events.length > 0) {
|
||||
sections.push({ heading: "What happened", lines: listWithMore(changes.events.map((e) => `- ${e.title}`)), attention: changes.events.some((e) => e.tone === "bad") });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read what happened:", err);
|
||||
}
|
||||
|
||||
// ── Disks heading for full ──
|
||||
try {
|
||||
const { diskForecastDays, diskUsagePercent } = settings.healthChecks;
|
||||
if (diskForecastDays > 0) {
|
||||
const lines: string[] = [];
|
||||
for (const s of serverRows) {
|
||||
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), diskForecastDays, diskUsagePercent)) {
|
||||
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
|
||||
}
|
||||
}
|
||||
if (lines.length > 0) sections.push({ heading: "Disks heading for full", lines: listWithMore(lines), attention: true });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the disk forecasts:", err);
|
||||
}
|
||||
|
||||
// ── About to expire ──
|
||||
try {
|
||||
const expiring: { daysLeft: number; text: string }[] = [];
|
||||
for (const s of await db.select().from(secrets)) {
|
||||
const status = computeSecretStatus(s.expiryDate, s.warnDays, now);
|
||||
if (status.status === "expired") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expired ${plural(Math.abs(status.daysLeft), "day")} ago` });
|
||||
else if (status.status === "expiring") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expires in ${plural(status.daysLeft, "day")} (${s.expiryDate})` });
|
||||
}
|
||||
for (const d of (await collectDomainAlerts(now)).expiring) {
|
||||
expiring.push({ daysLeft: d.daysLeft, text: d.daysLeft < 0 ? `Domain ${d.name} expired ${plural(Math.abs(d.daysLeft), "day")} ago` : `Domain ${d.name} expires in ${plural(d.daysLeft, "day")} (${d.expiresAt})` });
|
||||
}
|
||||
if (expiring.length > 0) {
|
||||
expiring.sort((a, b) => a.daysLeft - b.daysLeft);
|
||||
sections.push({ heading: "Expiring", lines: listWithMore(expiring.map((e) => `- ${e.text}`)), attention: true });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read what is expiring:", err);
|
||||
}
|
||||
|
||||
// ── Updates waiting ──
|
||||
try {
|
||||
const waiting = serverRows
|
||||
.map((s) => ({ name: s.name, updates: parsePendingUpdates(s.pendingUpdates), since: s.updatesPendingSince }))
|
||||
.filter((s) => s.updates && s.updates.count > 0);
|
||||
if (waiting.length > 0) {
|
||||
const security = waiting.reduce((n, s) => n + (s.updates!.security ?? 0), 0);
|
||||
const restarts = waiting.filter((s) => s.updates!.rebootRequired).length;
|
||||
const oldest = waiting.filter((s) => s.since).sort((a, b) => Date.parse(a.since!) - Date.parse(b.since!))[0];
|
||||
const lines = [`${plural(waiting.length, "server")} ${waiting.length === 1 ? "has" : "have"} updates waiting${security > 0 ? ` (${plural(security, "security fix", "security fixes")} among them)` : ""}.`];
|
||||
if (restarts > 0) lines.push(`${plural(restarts, "server")} ${restarts === 1 ? "needs" : "need"} a restart.`);
|
||||
if (oldest) lines.push(`Longest wait: ${oldest.name}, ${spanWords((now.getTime() - Date.parse(oldest.since!)) / 60_000)}.`);
|
||||
const withSecurity = waiting.filter((s) => (s.updates!.security ?? 0) > 0).sort((a, b) => (b.updates!.security ?? 0) - (a.updates!.security ?? 0));
|
||||
lines.push(...listWithMore(withSecurity.map((s) => `- ${s.name}: ${s.updates!.count} waiting, ${s.updates!.security} security`), 5));
|
||||
sections.push({ heading: "Updates waiting", lines, attention: security > 0 });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the pending updates:", err);
|
||||
}
|
||||
|
||||
// ── Planned ──
|
||||
try {
|
||||
const horizon = now.getTime() + DIGEST_DAYS * DAY_MS;
|
||||
const groups = new Map<string, { start: string; end: string; reason: string | null; names: string[] }>();
|
||||
for (const w of await listUpcomingWindows(now)) {
|
||||
if (Date.parse(w.startedAt) > horizon) continue;
|
||||
const target = await describeTarget(w.targetType, w.targetId);
|
||||
if (!target) continue;
|
||||
const key = `${w.startedAt}|${w.endsAt}|${w.reason ?? ""}`;
|
||||
const g = groups.get(key) ?? { start: w.startedAt, end: w.endsAt, reason: w.reason, names: [] };
|
||||
g.names.push(target.name);
|
||||
groups.set(key, g);
|
||||
}
|
||||
if (groups.size > 0) {
|
||||
const lines = [...groups.values()]
|
||||
.sort((a, b) => a.start.localeCompare(b.start))
|
||||
.map((g) => {
|
||||
const sorted = [...g.names].sort((a, b) => a.localeCompare(b));
|
||||
const names = sorted.length <= 3 ? sorted.join(", ") : `${sorted.slice(0, 3).join(", ")} and ${sorted.length - 3} more`;
|
||||
return `- ${whenWords(new Date(g.start), tz)}, ${spanWords((Date.parse(g.end) - Date.parse(g.start)) / 60_000)}: ${names}${g.reason ? ` — ${g.reason}` : ""}`;
|
||||
});
|
||||
sections.push({ heading: "Planned maintenance", lines: listWithMore(lines, 6), attention: false });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the planned maintenance:", err);
|
||||
}
|
||||
|
||||
const quiet = !sections.some((s) => s.attention);
|
||||
const envName = env.environment?.name;
|
||||
const title = `Homelab weekly digest${envName ? ` (${envName})` : ""}`;
|
||||
const body = sections.map((s) => `${s.heading.toUpperCase()}\n${s.lines.join("\n")}`).join("\n\n");
|
||||
const message = `${dayWords(now, tz)}\n${quiet ? "A quiet week: nothing needs attention.\n" : ""}\n${body}`.trim();
|
||||
return { title, message, quiet };
|
||||
}
|
||||
|
||||
/** Builds the digest and sends it through the channels — held with other notifications during quiet hours, unless `immediately`. */
|
||||
export async function sendDigest(options: { immediately?: boolean; now?: Date } = {}): Promise<Digest> {
|
||||
const digest = await buildDigest(options.now);
|
||||
if (options.immediately) await notifyNow(digest.title, digest.message);
|
||||
else await notify(digest.title, digest.message);
|
||||
return digest;
|
||||
}
|
||||
Reference in new issue
Block a user