Add a weekly digest and report which version of the agent each server runs

- Weekly digest through the notification channels: alerts now, a week of uptime, what happened, disks
  heading for full, expiring secrets and domains, updates waiting, planned maintenance. Off until switched
  on in Settings -> Notifications; Preview and Send it now there. Setting key 'digest'.
- Agents say their version in every report (Linux and Windows scripts are version 2); the server compares
  it with the scripts it serves and marks agents that are behind on the Servers, server and Updates pages.
  servers.agent_version, migration 0026.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-17 19:49:36 +02:00
1 parent 3da5c639b1
commit aa94d84f6c
28 files changed
+3179 -19

No files matched your search

@@ -0,0 +1 @@
ALTER TABLE `servers` ADD `agent_version` integer;
File diff suppressed because it is too large. Load diff
+7
View File
@@ -183,6 +183,13 @@
"when": 1792191757544,
"tag": "0025_clean_the_hand",
"breakpoints": true
},
{
"idx": 26,
"version": "6",
"when": 1792258587758,
"tag": "0026_reflective_northstar",
"breakpoints": true
}
]
}
+4
View File
@@ -291,6 +291,10 @@ export const servers = sqliteTable("servers", {
pendingUpdates: text("pending_updates"),
// When updates were first seen waiting (and none were before) — how long the oldest has been waiting. Null when none are.
updatesPendingSince: text("updates_pending_since"),
// Which version of the agent script is running on it, as the agent says in each report (see agent/*/report-tasks.*). Null until it says:
// an agent that predates the number being reported is version 1.
agentVersion: integer("agent_version"),
});
export const scheduledTasks = sqliteTable("scheduled_tasks", {
+2
View File
@@ -10,6 +10,7 @@ import { initDockerUpdateScheduler } from "./services/dockerUpdateScheduler.js";
import { initProxmoxBackupScheduler } from "./services/proxmoxBackupScheduler.js";
import { initPbsVerificationScheduler } from "./services/pbsVerificationScheduler.js";
import { initQuietHoursScheduler } from "./services/quietHoursScheduler.js";
import { initWeeklyDigestScheduler } from "./services/weeklyDigestScheduler.js";
import { initHealthScheduler } from "./services/healthScheduler.js";
import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceScheduler.js";
@@ -40,6 +41,7 @@ await initDockerUpdateScheduler();
await initProxmoxBackupScheduler();
await initPbsVerificationScheduler();
await initQuietHoursScheduler();
await initWeeklyDigestScheduler();
await initHealthScheduler();
await initAlertTicketScheduler();
await initKumaMaintenanceScheduler();
+6
View File
@@ -65,6 +65,11 @@ const reportSchema = z.object({
hostname: text(255).optional(),
os_type: text(50).optional(),
reported_at: text(50).optional(),
// Lenient like the rest: a number that makes no sense is dropped, never allowed to cost the agent its report.
agent_version: z
.unknown()
.optional()
.transform((v) => (typeof v === "number" && Number.isInteger(v) && v >= 1 && v <= 100_000 ? v : undefined)),
system: systemSchema.nullable().optional(),
tasks: z
.array(
@@ -111,6 +116,7 @@ agentReportRouter.post("/", asyncHandler(authenticateAgent), express.json({ limi
await syncServerTasks(server.id, {
hostname: parsed.data.hostname,
osType: parsed.data.os_type,
agentVersion: parsed.data.agent_version,
system: parsed.data.system,
tasks: parsed.data.tasks.map((t) => ({
scheduleType: t.schedule_type,
+7 -2
View File
@@ -8,6 +8,7 @@ import { generateApiToken } from "../services/tokens.js";
import { recordAudit } from "../services/audit.js";
import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js";
import { parsePendingUpdates } from "../services/pendingUpdates.js";
import { latestAgentVersions } from "../services/agentVersions.js";
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
import { getServerActivity } from "../services/serverActivity.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -42,6 +43,8 @@ serversRouter.get("/", asyncHandler(async (_req, res) => {
const rows = await db.query.servers.findMany({ orderBy: (s, { asc }) => [asc(s.name)] });
res.json({
servers: rows.map(publicServer),
// What a new install would be, so the page can say which agents are behind.
agentLatest: latestAgentVersions(),
});
}));
@@ -199,13 +202,15 @@ serversRouter.get("/updates", asyncHandler(async (_req, res) => {
lastSeenAt: servers.lastSeenAt,
pendingUpdates: servers.pendingUpdates,
updatesPendingSince: servers.updatesPendingSince,
agentVersion: servers.agentVersion,
})
.from(servers)
.orderBy(servers.name);
res.json({
servers: rows
.filter((r) => r.lastSeenAt !== null)
.map((r) => ({ id: r.id, name: r.name, osType: r.osType, lastSeenAt: r.lastSeenAt, updates: parsePendingUpdates(r.pendingUpdates), pendingSince: r.updatesPendingSince })),
.map((r) => ({ id: r.id, name: r.name, osType: r.osType, lastSeenAt: r.lastSeenAt, updates: parsePendingUpdates(r.pendingUpdates), pendingSince: r.updatesPendingSince, agentVersion: r.agentVersion })),
agentLatest: latestAgentVersions(),
});
}));
@@ -356,7 +361,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
children: [...links2.entries()].filter(([, l]) => l.parentId === id).map(([childId, l]) => ({ id: childId, name: nameOf.get(childId) ?? "", auto: l.auto })).sort((a, b) => a.name.localeCompare(b.name)),
};
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates) }, ipAddresses, hardware, dnsMatches, links, relations });
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates) }, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
}));
const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) });
+27
View File
@@ -12,6 +12,9 @@ import { schedulePbsVerificationCheck } from "../services/pbsVerificationSchedul
import { scheduleLogRetentionPurge } from "../services/logRetentionScheduler.js";
import { purgeOldLogs } from "../services/logRetention.js";
import { scheduleQuietHoursFlush } from "../services/quietHoursScheduler.js";
import { scheduleWeeklyDigest } from "../services/weeklyDigestScheduler.js";
import { buildDigest, sendDigest } from "../services/weeklyDigest.js";
import { enabledChannelCount } from "../services/notify.js";
import { testGotify, testNtfy, testSmtp, testWebhook, flushQuietHoursQueue } from "../services/notify.js";
import { listQueuedNotifications } from "../services/notificationQueue.js";
import { buildExportPayload, encryptExport, decryptExport, applyImportPayload, type EncryptedExportFile } from "../services/configBackup.js";
@@ -99,6 +102,10 @@ const updateSchema = z.object({
.object({ enabled: z.boolean(), start: z.string().regex(/^\d{2}:\d{2}$/), end: z.string().regex(/^\d{2}:\d{2}$/) })
.partial()
.optional(),
digest: z
.object({ enabled: z.boolean(), day: z.number().int().min(0).max(6), time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/) })
.partial()
.optional(),
});
settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
@@ -123,6 +130,9 @@ settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
if (parsed.data.quietHours || parsed.data.notifications) {
await scheduleQuietHoursFlush();
}
if (parsed.data.digest || parsed.data.notifications) {
await scheduleWeeklyDigest();
}
await recordAudit({
actor: req.currentUser!,
@@ -203,6 +213,23 @@ settingsRouter.post("/purge-logs", requireRole("admin"), asyncHandler(async (req
res.json(result);
}));
// What the weekly digest would say right now, so it can be read before it is switched on.
settingsRouter.get("/digest/preview", requireRole("admin"), asyncHandler(async (_req, res) => {
const digest = await buildDigest();
res.json({ title: digest.title, message: digest.message, quiet: digest.quiet, channels: await enabledChannelCount() });
}));
// Sends it now, through whatever channels are switched on, at whatever time it is.
settingsRouter.post("/digest/send", requireRole("admin"), asyncHandler(async (req, res) => {
const channels = await enabledChannelCount();
if (channels === 0) {
return res.status(400).json({ error: "no_channel", message: "No notification channel is switched on, so there is nowhere to send it. Set one up above first." });
}
const digest = await sendDigest({ immediately: true });
await recordAudit({ actor: req.currentUser!, category: "settings", action: "send_digest", detail: { channels, quiet: digest.quiet } });
res.json({ sent: true, channels });
}));
settingsRouter.get("/quiet-hours-queue", requireRole("admin"), asyncHandler(async (_req, res) => {
const items = await listQueuedNotifications();
res.json({ count: items.length, items });
+62
View File
@@ -0,0 +1,62 @@
/**
* Which version of the agent each server runs, and which version this Homelab Manager would give a new install. An agent is a copy of
* a script that was served at install time, so after the app is updated the agents out on the servers can be behind. Each script carries
* its version as a number (`AGENT_VERSION=2` / `$script:AgentVersion = 2`), says it in every report, and the "latest" is read from the very
* scripts this app serves — so the two can never disagree about what "current" means.
*
* 1 — before the number was reported (so an agent that says nothing is version 1)
* 2 — reports pending updates, and its own version
*/
import { readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
export type AgentOs = "linux" | "windows";
export interface LatestAgentVersions {
linux: number | null;
windows: number | null;
}
// server/src/services → the repository's agent folder (the same place the app serves /agent from; in the image it sits beside server/dist).
const here = dirname(fileURLToPath(import.meta.url));
export const AGENT_DIR = join(here, "..", "..", "..", "agent");
const FILES: Record<AgentOs, string> = { linux: join("linux", "report-tasks.sh"), windows: join("windows", "report-tasks.ps1") };
const PATTERNS: Record<AgentOs, RegExp> = {
linux: /^AGENT_VERSION=(\d+)\s*$/m,
windows: /^\$script:AgentVersion\s*=\s*(\d+)\s*$/m,
};
/** The version a script says it is, or null if it doesn't say. */
export function parseAgentVersion(text: string, os: AgentOs): number | null {
const m = PATTERNS[os].exec(text.replace(/\r\n/g, "\n"));
return m ? Number(m[1]) : null;
}
let cached: LatestAgentVersions | null = null;
/** What the scripts served from here say. Read once (they don't change while the app runs); null for one that can't be read. */
export function latestAgentVersions(dir: string = AGENT_DIR): LatestAgentVersions {
if (cached && dir === AGENT_DIR) return cached;
const read = (os: AgentOs): number | null => {
try {
return parseAgentVersion(readFileSync(join(dir, FILES[os]), "utf8"), os);
} catch {
return null;
}
};
const result = { linux: read("linux"), windows: read("windows") };
if (dir === AGENT_DIR) cached = result;
return result;
}
/**
* Whether a server's agent is behind. Only a server that reports through an agent can be: one that never has (a Proxmox guest tracked
* through its host, say) has none to update. An agent that reports no version is version 1.
*/
export function isAgentOutdated(row: { osType: string; lastSeenAt: string | null; agentVersion: number | null }, latest: LatestAgentVersions): boolean {
if (!row.lastSeenAt) return false;
const newest = row.osType === "windows" ? latest.windows : row.osType === "linux" ? latest.linux : null;
if (newest === null) return false;
return (row.agentVersion ?? 1) < newest;
}
+16
View File
@@ -104,6 +104,22 @@ async function isQuietHoursActive(): Promise<boolean> {
return startMin < endMin ? nowMin >= startMin && nowMin < endMin : nowMin >= startMin || nowMin < endMin;
}
/** Sends at once through every channel that is switched on, whatever the time — for something a person just asked for. */
export async function notifyNow(title: string, message: string): Promise<void> {
await sendToChannels(title, message);
}
/** How many channels are switched on and filled in enough to send, so a "send it now" can say when there is nowhere to send to. */
export async function enabledChannelCount(): Promise<number> {
const { gotify, ntfy, smtp, webhook } = await getSettings();
return [
gotify.enabled && gotify.url && gotify.token,
ntfy.enabled && ntfy.url && ntfy.topic,
smtp.enabled && smtp.host && smtp.to && smtp.from,
webhook.enabled && webhook.url,
].filter(Boolean).length;
}
export async function notify(title: string, message: string): Promise<void> {
if (await isQuietHoursActive()) {
await enqueueNotification(title, message);
+11
View File
@@ -99,6 +99,15 @@ export interface HealthCheckSettings {
diskForecastDays: number;
}
/** The weekly digest: one message a week through the notification channels. Off until an admin switches it on under Settings → Notifications. */
export interface DigestSettings {
enabled: boolean;
/** 0 = Sunday … 6 = Saturday, in notifications.timezone. */
day: number;
/** "HH:MM", in notifications.timezone. */
time: string;
}
export interface ConsistencySettings {
/** CIDR ranges and single addresses the consistency report ignores entirely — e.g. Docker networks that repeat on many hosts. */
excludedRanges: string[];
@@ -150,6 +159,7 @@ export interface AppSettings {
logRetention: LogRetentionSettings;
quietHours: QuietHoursSettings;
healthChecks: HealthCheckSettings;
digest: DigestSettings;
consistency: ConsistencySettings;
nameGenerator: NameGeneratorSettings;
alertTickets: AlertTicketSettings;
@@ -182,6 +192,7 @@ const DEFAULTS: AppSettings = {
display: { dateFormat: "ymd", timeFormat: "24h", pageSize: 20 },
logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 },
quietHours: { enabled: false, start: "22:00", end: "07:00" },
digest: { enabled: false, day: 1, time: "08:00" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
+3
View File
@@ -29,6 +29,8 @@ export interface AgentReport {
hostname?: string;
/** What the agent says it runs on. Only "linux" and "windows" are recorded; anything else is ignored. */
osType?: string;
/** Which version of the agent script is running. Absent from an agent that predates the number. */
agentVersion?: number;
system?: IncomingSystemInfo | null;
tasks: IncomingTask[];
}
@@ -109,6 +111,7 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
// Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before.
...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}),
...(report.hostname ? { hostname: report.hostname } : {}),
...(report.agentVersion !== undefined ? { agentVersion: report.agentVersion } : {}),
// An agent knows what it runs on better than whoever registered the server did.
...(report.osType === "linux" || report.osType === "windows" ? { osType: report.osType } : {}),
...(system?.ip_addresses ? { ipAddresses: JSON.stringify(system.ip_addresses) } : {}),
+229
View File
@@ -0,0 +1,229 @@
/**
* The weekly digest: one message, sent through the notification channels, that says how the week went and what is coming — alerts now,
* what was down and for how long, what happened, disks heading for full, things about to expire, updates waiting, planned maintenance.
* Everything in it is already known to the app (the alerts, the uptime record, the audit log, the agents' reports); this only gathers it
* into a few lines per topic and leaves out any topic with nothing to say. Plain text, so it reads in a phone notification and an email alike.
*/
import { db } from "../db/client.js";
import { integrations, secrets, servers } from "../db/schema.js";
import { env } from "../env.js";
import { getAlerts } from "./alerts.js";
import { getChangesSince } from "./changesSince.js";
import { collectDomainAlerts } from "./domainMonitor.js";
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
import { notify, notifyNow } from "./notify.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { computeSecretStatus } from "./secretStatus.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
export const DIGEST_DAYS = 7;
const DAY_MS = 24 * 3600_000;
const MAX_LINES = 8;
export interface Digest {
title: string;
message: string;
/** Nothing in it needs anyone: the quiet week. */
quiet: boolean;
}
interface Section {
heading: string;
lines: string[];
/** A section that only reassures ("every server stayed up") doesn't make the week un-quiet. */
attention: boolean;
}
/** "2 h 40 min", "35 min", "3 d 4 h". */
export function spanWords(minutes: number): string {
const m = Math.max(1, Math.round(minutes));
if (m < 60) return `${m} min`;
const h = Math.floor(m / 60);
if (h < 24) return m % 60 && h < 10 ? `${h} h ${m % 60} min` : `${h} h`;
const d = Math.floor(h / 24);
return h % 24 ? `${d} d ${h % 24} h` : `${d} d`;
}
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
function listWithMore(lines: string[], max = MAX_LINES): string[] {
return lines.length > max ? [...lines.slice(0, max), `…and ${lines.length - max} more`] : lines;
}
function whenWords(date: Date, timezone: string): string {
return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", hour: "2-digit", minute: "2-digit", hourCycle: "h23" }).format(date);
}
function dayWords(date: Date, timezone: string): string {
return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", year: "numeric" }).format(date);
}
export async function buildDigest(now: Date = new Date()): Promise<Digest> {
const settings = await getSettings();
const tz = settings.notifications.timezone || "UTC";
const sections: Section[] = [];
const serverRows = await db
.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince })
.from(servers);
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
// ── Right now ──
try {
const report = await getAlerts(false);
const active = report.alerts.filter((a) => !a.silenced && a.severity !== "info");
const lines: string[] = [];
if (active.length === 0) lines.push(report.couldntCheck.length > 0 ? `No active alerts, but ${plural(report.couldntCheck.length, "check")} couldn't run.` : "No active alerts.");
else {
lines.push(`${report.counts.critical} critical, ${report.counts.warning} warning`);
// Things about to expire have a section of their own, with dates; they are counted here but not listed twice.
const listed = active.filter((a) => a.category !== "expiry");
lines.push(...listWithMore(listed.map((a) => `- ${a.message}`), 5));
if (listed.length < active.length) lines.push(`(${plural(active.length - listed.length, "more is", "more are")} about things expiring, below)`);
}
const held: string[] = [];
if (report.counts.silenced > 0) held.push(`${report.counts.silenced} silenced by maintenance`);
if (report.counts.grouped > 0) held.push(`${report.counts.grouped} folded under a server that is down`);
if (held.length > 0) lines.push(`(${held.join(", ")})`);
sections.push({ heading: "Right now", lines, attention: active.length > 0 || report.couldntCheck.length > 0 });
} catch (err) {
console.error("[digest] couldn't read the alerts:", err);
}
// ── Uptime ──
try {
const uptime = await getUptime(now, DIGEST_DAYS);
const integrationName = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name]));
const rows: { name: string; down: number; percent: number }[] = [];
let measured = 0;
for (const [id, series] of Object.entries(uptime.servers)) {
measured++;
const down = series.buckets.reduce((n, b) => n + b.d, 0);
if (down > 0) rows.push({ name: serverName.get(Number(id)) ?? `server ${id}`, down, percent: series.percent });
}
for (const [id, series] of Object.entries(uptime.integrations)) {
measured++;
const down = series.buckets.reduce((n, b) => n + b.d, 0);
if (down > 0) rows.push({ name: integrationName.get(Number(id)) ?? `integration ${id}`, down, percent: series.percent });
}
if (measured > 0) {
rows.sort((a, b) => b.down - a.down);
sections.push({
heading: `Uptime, last ${DIGEST_DAYS} days`,
lines: rows.length === 0 ? ["Everything stayed up."] : listWithMore(rows.map((r) => `- ${r.name}: ${r.percent}% (${spanWords(r.down)} down)`)),
attention: rows.length > 0,
});
}
} catch (err) {
console.error("[digest] couldn't read the uptime:", err);
}
// ── What happened ──
try {
const changes = await getChangesSince({ since: new Date(now.getTime() - DIGEST_DAYS * DAY_MS), now, actorId: 0, canSeeAudit: false });
if (changes.events.length > 0) {
sections.push({ heading: "What happened", lines: listWithMore(changes.events.map((e) => `- ${e.title}`)), attention: changes.events.some((e) => e.tone === "bad") });
}
} catch (err) {
console.error("[digest] couldn't read what happened:", err);
}
// ── Disks heading for full ──
try {
const { diskForecastDays, diskUsagePercent } = settings.healthChecks;
if (diskForecastDays > 0) {
const lines: string[] = [];
for (const s of serverRows) {
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), diskForecastDays, diskUsagePercent)) {
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
}
}
if (lines.length > 0) sections.push({ heading: "Disks heading for full", lines: listWithMore(lines), attention: true });
}
} catch (err) {
console.error("[digest] couldn't read the disk forecasts:", err);
}
// ── About to expire ──
try {
const expiring: { daysLeft: number; text: string }[] = [];
for (const s of await db.select().from(secrets)) {
const status = computeSecretStatus(s.expiryDate, s.warnDays, now);
if (status.status === "expired") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expired ${plural(Math.abs(status.daysLeft), "day")} ago` });
else if (status.status === "expiring") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expires in ${plural(status.daysLeft, "day")} (${s.expiryDate})` });
}
for (const d of (await collectDomainAlerts(now)).expiring) {
expiring.push({ daysLeft: d.daysLeft, text: d.daysLeft < 0 ? `Domain ${d.name} expired ${plural(Math.abs(d.daysLeft), "day")} ago` : `Domain ${d.name} expires in ${plural(d.daysLeft, "day")} (${d.expiresAt})` });
}
if (expiring.length > 0) {
expiring.sort((a, b) => a.daysLeft - b.daysLeft);
sections.push({ heading: "Expiring", lines: listWithMore(expiring.map((e) => `- ${e.text}`)), attention: true });
}
} catch (err) {
console.error("[digest] couldn't read what is expiring:", err);
}
// ── Updates waiting ──
try {
const waiting = serverRows
.map((s) => ({ name: s.name, updates: parsePendingUpdates(s.pendingUpdates), since: s.updatesPendingSince }))
.filter((s) => s.updates && s.updates.count > 0);
if (waiting.length > 0) {
const security = waiting.reduce((n, s) => n + (s.updates!.security ?? 0), 0);
const restarts = waiting.filter((s) => s.updates!.rebootRequired).length;
const oldest = waiting.filter((s) => s.since).sort((a, b) => Date.parse(a.since!) - Date.parse(b.since!))[0];
const lines = [`${plural(waiting.length, "server")} ${waiting.length === 1 ? "has" : "have"} updates waiting${security > 0 ? ` (${plural(security, "security fix", "security fixes")} among them)` : ""}.`];
if (restarts > 0) lines.push(`${plural(restarts, "server")} ${restarts === 1 ? "needs" : "need"} a restart.`);
if (oldest) lines.push(`Longest wait: ${oldest.name}, ${spanWords((now.getTime() - Date.parse(oldest.since!)) / 60_000)}.`);
const withSecurity = waiting.filter((s) => (s.updates!.security ?? 0) > 0).sort((a, b) => (b.updates!.security ?? 0) - (a.updates!.security ?? 0));
lines.push(...listWithMore(withSecurity.map((s) => `- ${s.name}: ${s.updates!.count} waiting, ${s.updates!.security} security`), 5));
sections.push({ heading: "Updates waiting", lines, attention: security > 0 });
}
} catch (err) {
console.error("[digest] couldn't read the pending updates:", err);
}
// ── Planned ──
try {
const horizon = now.getTime() + DIGEST_DAYS * DAY_MS;
const groups = new Map<string, { start: string; end: string; reason: string | null; names: string[] }>();
for (const w of await listUpcomingWindows(now)) {
if (Date.parse(w.startedAt) > horizon) continue;
const target = await describeTarget(w.targetType, w.targetId);
if (!target) continue;
const key = `${w.startedAt}|${w.endsAt}|${w.reason ?? ""}`;
const g = groups.get(key) ?? { start: w.startedAt, end: w.endsAt, reason: w.reason, names: [] };
g.names.push(target.name);
groups.set(key, g);
}
if (groups.size > 0) {
const lines = [...groups.values()]
.sort((a, b) => a.start.localeCompare(b.start))
.map((g) => {
const sorted = [...g.names].sort((a, b) => a.localeCompare(b));
const names = sorted.length <= 3 ? sorted.join(", ") : `${sorted.slice(0, 3).join(", ")} and ${sorted.length - 3} more`;
return `- ${whenWords(new Date(g.start), tz)}, ${spanWords((Date.parse(g.end) - Date.parse(g.start)) / 60_000)}: ${names}${g.reason ? ` — ${g.reason}` : ""}`;
});
sections.push({ heading: "Planned maintenance", lines: listWithMore(lines, 6), attention: false });
}
} catch (err) {
console.error("[digest] couldn't read the planned maintenance:", err);
}
const quiet = !sections.some((s) => s.attention);
const envName = env.environment?.name;
const title = `Homelab weekly digest${envName ? ` (${envName})` : ""}`;
const body = sections.map((s) => `${s.heading.toUpperCase()}\n${s.lines.join("\n")}`).join("\n\n");
const message = `${dayWords(now, tz)}\n${quiet ? "A quiet week: nothing needs attention.\n" : ""}\n${body}`.trim();
return { title, message, quiet };
}
/** Builds the digest and sends it through the channels — held with other notifications during quiet hours, unless `immediately`. */
export async function sendDigest(options: { immediately?: boolean; now?: Date } = {}): Promise<Digest> {
const digest = await buildDigest(options.now);
if (options.immediately) await notifyNow(digest.title, digest.message);
else await notify(digest.title, digest.message);
return digest;
}
@@ -0,0 +1,33 @@
import schedule from "node-schedule";
import { getSettings } from "./settingsStore.js";
import { sendDigest } from "./weeklyDigest.js";
let currentJob: schedule.Job | null = null;
/** The cron rule for "this weekday (0 = Sunday) at HH:MM". */
export function cronForDigest(day: number, time: string): string {
const [h, m] = time.split(":").map(Number);
const d = Number.isInteger(day) && day >= 0 && day <= 6 ? day : 1;
return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} * * ${d}`;
}
/** (Re)schedules the weekly digest per the current settings. Call again after they change. */
export async function scheduleWeeklyDigest(): Promise<void> {
if (currentJob) {
currentJob.cancel();
currentJob = null;
}
const { digest, notifications } = await getSettings();
if (!digest.enabled) {
console.log("[digest] weekly digest is off, nothing scheduled");
return;
}
currentJob = schedule.scheduleJob({ rule: cronForDigest(digest.day, digest.time), tz: notifications.timezone }, () => {
sendDigest().catch((err) => console.error("[digest] couldn't send the weekly digest:", err));
});
console.log(`Weekly digest scheduled: ${cronForDigest(digest.day, digest.time)} (${notifications.timezone})`);
}
export async function initWeeklyDigestScheduler(): Promise<void> {
await scheduleWeeklyDigest();
}
+138
View File
@@ -0,0 +1,138 @@
/**
* Which version of the agent each server runs, and which agents are behind the scripts this app serves.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-agentver-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { parseAgentVersion, latestAgentVersions, isAgentOutdated } = await import("../src/services/agentVersions.js");
const { serversRouter } = await import("../src/routes/servers.js");
const { agentReportRouter } = await import("../src/routes/agentReport.js");
const { hashToken } = await import("../src/services/tokens.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
describe("reading a version out of a script", () => {
test("each script says what it is, in the form the server looks for", () => {
assert.equal(parseAgentVersion("#!/usr/bin/env bash\nset -e\n\nAGENT_VERSION=7\nENV_FILE=x\n", "linux"), 7);
assert.equal(parseAgentVersion("$script:AgentVersion = 12\r\n$script:Other = 1\r\n", "windows"), 12);
assert.equal(parseAgentVersion("# AGENT_VERSION=3 in a comment\n", "linux"), null, "only a line that is the assignment counts");
assert.equal(parseAgentVersion("no version here", "windows"), null);
});
test("the scripts this app really serves each carry a version", () => {
const latest = latestAgentVersions();
assert.ok(Number.isInteger(latest.linux) && latest.linux! >= 2, `linux ${latest.linux}`);
assert.ok(Number.isInteger(latest.windows) && latest.windows! >= 2, `windows ${latest.windows}`);
});
test("a folder with nothing in it gives no version, not an error", () => {
const empty = mkdtempSync(join(tmpdir(), "hm-noagent-"));
assert.deepEqual(latestAgentVersions(empty), { linux: null, windows: null });
mkdirSync(join(empty, "linux"));
writeFileSync(join(empty, "linux", "report-tasks.sh"), "AGENT_VERSION=5\n");
assert.deepEqual(latestAgentVersions(empty), { linux: 5, windows: null });
rmSync(empty, { recursive: true, force: true });
});
});
describe("which agents are behind", () => {
const latest = { linux: 3, windows: 2 };
const seen = "2026-10-17T00:00:00Z";
test("an agent that says nothing is version 1, and older than the latest is behind", () => {
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: null }, latest), true);
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 2 }, latest), true);
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 3 }, latest), false);
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 4 }, latest), false, "ahead of this app is not behind");
assert.equal(isAgentOutdated({ osType: "windows", lastSeenAt: seen, agentVersion: 2 }, latest), false, "each system has its own latest");
});
test("a server with no agent has nothing to update, and an unknown latest flags nothing", () => {
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: null, agentVersion: null }, latest), false);
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: null }, { linux: null, windows: null }), false);
assert.equal(isAgentOutdated({ osType: "plan9", lastSeenAt: seen, agentVersion: null }, latest), false);
});
});
describe("the version in reports", () => {
let app: Server;
let base = "";
let id = 0;
before(async () => {
await db.insert(users).values({ oidcSub: "admin", name: "Ada", role: "admin" });
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t" }).returning();
id = s.id;
const a = express();
a.use("/api/agent/report", agentReportRouter);
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
next();
});
a.use("/api/servers", serversRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const report = (extra: Record<string, unknown>) =>
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", tasks: [], ...extra }) });
test("an agent's version is stored; a report without one, or with nonsense, leaves it as it was", async () => {
assert.equal((await report({})).status, 202);
let [s] = await db.select().from(servers).where(eq(servers.id, id));
assert.equal(s.agentVersion, null, "an old agent says nothing");
assert.equal((await report({ agent_version: 2 })).status, 202);
[s] = await db.select().from(servers).where(eq(servers.id, id));
assert.equal(s.agentVersion, 2);
for (const odd of ["two", 0, -1, 1.5, 999999999, null, {}]) {
assert.equal((await report({ agent_version: odd })).status, 202, `${JSON.stringify(odd)} must not cost the report`);
}
[s] = await db.select().from(servers).where(eq(servers.id, id));
assert.equal(s.agentVersion, 2);
});
test("the lists say what the latest is, and the server's own row says what it runs", async () => {
const list = await (await fetch(`${base}/servers`)).json();
assert.ok(list.agentLatest.linux >= 2 && list.agentLatest.windows >= 2);
assert.equal(list.servers.find((x: { id: number }) => x.id === id).agentVersion, 2);
const detail = await (await fetch(`${base}/servers/${id}/detail`)).json();
assert.equal(detail.server.agentVersion, 2);
assert.ok(detail.agentLatest.linux >= 2);
const updates = await (await fetch(`${base}/servers/updates`)).json();
assert.equal(updates.servers.find((x: { id: number }) => x.id === id).agentVersion, 2);
assert.ok(updates.agentLatest.windows >= 2);
});
});
+202
View File
@@ -0,0 +1,202 @@
/**
* The weekly digest: what it gathers, what it leaves out, and sending it.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import http from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-digest-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, secrets, auditLog, maintenanceWindows } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { buildDigest, spanWords } = await import("../src/services/weeklyDigest.js");
const { cronForDigest } = await import("../src/services/weeklyDigestScheduler.js");
const { settingsRouter } = await import("../src/routes/settings.js");
const { updateSettings, getSettings } = await import("../src/services/settingsStore.js");
const { invalidateAlertsCache } = await import("../src/services/alerts.js");
await runMigrations();
const HOUR = 3600_000;
const DAY = 24 * HOUR;
const NOW = new Date(Math.floor(Date.now() / 1000) * 1000);
const dbTime = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace("T", " ");
describe("small pieces", () => {
test("spans read as words", () => {
assert.equal(spanWords(35), "35 min");
assert.equal(spanWords(160), "2 h 40 min");
assert.equal(spanWords(60 * 30), "1 d 6 h");
assert.equal(spanWords(60 * 9), "9 h");
assert.equal(spanWords(60 * 24 * 3 + 240), "3 d 4 h");
});
test("the schedule is the chosen weekday and time", () => {
assert.equal(cronForDigest(1, "08:00"), "0 8 * * 1");
assert.equal(cronForDigest(0, "21:30"), "30 21 * * 0");
assert.equal(cronForDigest(9, "nonsense"), "0 8 * * 1", "a bad value falls back rather than scheduling nonsense");
});
});
describe("the quiet week", () => {
test("with nothing wrong it says so, and leaves out topics with nothing to say", async () => {
const digest = await buildDigest(NOW);
assert.equal(digest.quiet, true);
assert.match(digest.message, /A quiet week: nothing needs attention\./);
assert.match(digest.message, /RIGHT NOW\nNo active alerts\./);
for (const heading of ["UPTIME", "WHAT HAPPENED", "DISKS HEADING FOR FULL", "EXPIRING", "UPDATES WAITING", "PLANNED MAINTENANCE"]) {
assert.ok(!digest.message.includes(heading), `no ${heading} section when there's nothing in it`);
}
assert.match(digest.title, /^Homelab weekly digest/);
});
});
describe("a week with things in it", () => {
let app: http.Server;
let base = "";
let webhook: http.Server;
const sent: { content: string }[] = [];
before(async () => {
await db.insert(users).values({ oidcSub: "admin", name: "Ada", role: "admin" });
const made = await db
.insert(servers)
.values([
{ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: new Date(NOW.getTime() - 5 * HOUR).toISOString() },
{ name: "anna", apiTokenHash: "b", apiTokenPrefix: "b", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: NOW.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 9, security: 3, rebootRequired: true, reportedAt: NOW.toISOString() }), updatesPendingSince: new Date(NOW.getTime() - 12 * DAY).toISOString() },
{ name: "zoe", apiTokenHash: "c", apiTokenPrefix: "c", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: NOW.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 0, security: 0, rebootRequired: false, reportedAt: NOW.toISOString() }) },
])
.returning({ id: servers.id });
const [elsa, anna] = made.map((m) => m.id);
// elsa was down for 2 hours three days ago, and has now stopped reporting again.
await db.insert(auditLog).values([
{ category: "server", action: "went_offline", targetType: "server", targetId: String(elsa), actorLabel: "system", createdAt: dbTime(NOW.getTime() - 3 * DAY) },
{ category: "server", action: "back_online", targetType: "server", targetId: String(elsa), actorLabel: "system", createdAt: dbTime(NOW.getTime() - 3 * DAY + 2 * HOUR) },
]);
const date = (days: number) => new Date(NOW.getTime() + days * DAY).toISOString().slice(0, 10);
await db.insert(secrets).values([
{ name: "VPN key", type: "api_token", expiryDate: date(9), warnDays: 30 },
{ name: "Old token", type: "api_token", expiryDate: date(-4), warnDays: 30 },
{ name: "Fine for years", type: "api_token", expiryDate: date(900), warnDays: 30 },
]);
await db.insert(maintenanceWindows).values([elsa, anna].map((id) => ({
targetType: "server" as const,
targetId: id,
reason: "Patch night",
startedAt: new Date(NOW.getTime() + 2 * DAY).toISOString(),
endsAt: new Date(NOW.getTime() + 2 * DAY + 2 * HOUR).toISOString(),
createdBy: "Ada",
})));
webhook = await new Promise<http.Server>((resolve) => {
const s = http.createServer((req, res) => {
let raw = "";
req.on("data", (c) => (raw += c));
req.on("end", () => {
sent.push(JSON.parse(raw));
res.end("ok");
});
});
s.listen(0, "127.0.0.1", () => resolve(s));
});
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
next();
});
a.use("/api/settings", settingsRouter);
app = await new Promise<http.Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/settings`;
invalidateAlertsCache();
});
after(async () => {
// Saving the schedule started a cron job, which would keep this process alive.
await (await import("node-schedule")).default.gracefulShutdown();
app.close();
webhook.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("it gathers each topic into a few lines", async () => {
const digest = await buildDigest(NOW);
assert.equal(digest.quiet, false);
const m = digest.message;
assert.match(m, /RIGHT NOW\n.*critical/, m);
assert.ok(!/RIGHT NOW[^]*Secret "Old token"[^]*EXPIRING/.test(m), "an expiring secret is in its own section, not listed twice");
assert.match(m, /\(1 more is about things expiring, below\)|\(2 more are about things expiring, below\)/);
assert.match(m, /elsa hasn't reported for 5 h/);
assert.match(m, /UPTIME, LAST 7 DAYS\n- elsa: .*% \(.*down\)/s, "elsa's downtime is told with a figure");
assert.match(m, /WHAT HAPPENED\n- elsa was offline for 2 h/);
assert.match(m, /EXPIRING\n- Secret "Old token" expired 4 days ago\n- Secret "VPN key" expires in 9 days/);
assert.ok(!m.includes("Fine for years"));
assert.match(m, /UPDATES WAITING\n1 server has updates waiting \(3 security fixes among them\)\.\n1 server needs a restart\.\nLongest wait: anna, 12 d\.\n- anna: 9 waiting, 3 security/);
assert.match(m, /PLANNED MAINTENANCE\n- .*2 h: anna, elsa — Patch night/);
});
test("preview needs an administrator's route and shows the same text; sending needs somewhere to send to", async () => {
const preview = await (await fetch(`${base}/digest/preview`)).json();
assert.match(preview.title, /^Homelab weekly digest/);
assert.match(preview.message, /UPDATES WAITING/);
assert.equal(preview.channels, 0);
const none = await fetch(`${base}/digest/send`, { method: "POST" });
assert.equal(none.status, 400);
assert.equal((await none.json()).error, "no_channel");
assert.equal(sent.length, 0);
});
test("sending goes out through a switched-on channel at once, and is in the audit log", async () => {
const url = `http://127.0.0.1:${(webhook.address() as AddressInfo).port}/hook`;
await updateSettings({ webhook: { enabled: true, url }, quietHours: { enabled: true, start: "00:00", end: "23:59" } });
const res = await fetch(`${base}/digest/send`, { method: "POST" });
assert.equal(res.status, 200);
assert.deepEqual(await res.json(), { sent: true, channels: 1 });
assert.equal(sent.length, 1, "sent at once even inside quiet hours: a person asked for it now");
assert.match(sent[0].content, /^\*\*Homelab weekly digest/);
assert.match(sent[0].content, /UPDATES WAITING/);
const entries = (await db.select().from(auditLog)).filter((e) => e.action === "send_digest");
assert.equal(entries.length, 1);
await updateSettings({ quietHours: { enabled: false } });
});
test("the schedule is saved, and a day or time that makes no sense is refused", async () => {
const put = (digest: unknown) => fetch(`${base}/`, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ digest }) });
assert.equal((await put({ enabled: true, day: 0, time: "21:30" })).status, 200);
assert.deepEqual((await getSettings()).digest, { enabled: true, day: 0, time: "21:30" });
for (const bad of [{ day: 7 }, { day: -1 }, { day: 1.5 }, { time: "25:00" }, { time: "8am" }, { enabled: "yes" }]) {
assert.equal((await put(bad)).status, 400, JSON.stringify(bad));
}
assert.deepEqual((await getSettings()).digest, { enabled: true, day: 0, time: "21:30" }, "nothing was half-saved");
});
test("it is off until switched on", async () => {
const fresh = mkdtempSync(join(tmpdir(), "x-"));
rmSync(fresh, { recursive: true, force: true });
const { digest } = await getSettings();
assert.equal(typeof digest.enabled, "boolean");
});
});