Add a weekly digest and report which version of the agent each server runs

- Weekly digest through the notification channels: alerts now, a week of uptime, what happened, disks
  heading for full, expiring secrets and domains, updates waiting, planned maintenance. Off until switched
  on in Settings -> Notifications; Preview and Send it now there. Setting key 'digest'.
- Agents say their version in every report (Linux and Windows scripts are version 2); the server compares
  it with the scripts it serves and marks agents that are behind on the Servers, server and Updates pages.
  servers.agent_version, migration 0026.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-17 19:49:36 +02:00
1 parent 3da5c639b1
commit aa94d84f6c
28 files changed
+3179 -19

No files matched your search

+202
View File
@@ -0,0 +1,202 @@
/**
* The weekly digest: what it gathers, what it leaves out, and sending it.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import http from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-digest-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, secrets, auditLog, maintenanceWindows } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { buildDigest, spanWords } = await import("../src/services/weeklyDigest.js");
const { cronForDigest } = await import("../src/services/weeklyDigestScheduler.js");
const { settingsRouter } = await import("../src/routes/settings.js");
const { updateSettings, getSettings } = await import("../src/services/settingsStore.js");
const { invalidateAlertsCache } = await import("../src/services/alerts.js");
await runMigrations();
const HOUR = 3600_000;
const DAY = 24 * HOUR;
const NOW = new Date(Math.floor(Date.now() / 1000) * 1000);
const dbTime = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace("T", " ");
describe("small pieces", () => {
test("spans read as words", () => {
assert.equal(spanWords(35), "35 min");
assert.equal(spanWords(160), "2 h 40 min");
assert.equal(spanWords(60 * 30), "1 d 6 h");
assert.equal(spanWords(60 * 9), "9 h");
assert.equal(spanWords(60 * 24 * 3 + 240), "3 d 4 h");
});
test("the schedule is the chosen weekday and time", () => {
assert.equal(cronForDigest(1, "08:00"), "0 8 * * 1");
assert.equal(cronForDigest(0, "21:30"), "30 21 * * 0");
assert.equal(cronForDigest(9, "nonsense"), "0 8 * * 1", "a bad value falls back rather than scheduling nonsense");
});
});
describe("the quiet week", () => {
test("with nothing wrong it says so, and leaves out topics with nothing to say", async () => {
const digest = await buildDigest(NOW);
assert.equal(digest.quiet, true);
assert.match(digest.message, /A quiet week: nothing needs attention\./);
assert.match(digest.message, /RIGHT NOW\nNo active alerts\./);
for (const heading of ["UPTIME", "WHAT HAPPENED", "DISKS HEADING FOR FULL", "EXPIRING", "UPDATES WAITING", "PLANNED MAINTENANCE"]) {
assert.ok(!digest.message.includes(heading), `no ${heading} section when there's nothing in it`);
}
assert.match(digest.title, /^Homelab weekly digest/);
});
});
describe("a week with things in it", () => {
let app: http.Server;
let base = "";
let webhook: http.Server;
const sent: { content: string }[] = [];
before(async () => {
await db.insert(users).values({ oidcSub: "admin", name: "Ada", role: "admin" });
const made = await db
.insert(servers)
.values([
{ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: new Date(NOW.getTime() - 5 * HOUR).toISOString() },
{ name: "anna", apiTokenHash: "b", apiTokenPrefix: "b", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: NOW.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 9, security: 3, rebootRequired: true, reportedAt: NOW.toISOString() }), updatesPendingSince: new Date(NOW.getTime() - 12 * DAY).toISOString() },
{ name: "zoe", apiTokenHash: "c", apiTokenPrefix: "c", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: NOW.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 0, security: 0, rebootRequired: false, reportedAt: NOW.toISOString() }) },
])
.returning({ id: servers.id });
const [elsa, anna] = made.map((m) => m.id);
// elsa was down for 2 hours three days ago, and has now stopped reporting again.
await db.insert(auditLog).values([
{ category: "server", action: "went_offline", targetType: "server", targetId: String(elsa), actorLabel: "system", createdAt: dbTime(NOW.getTime() - 3 * DAY) },
{ category: "server", action: "back_online", targetType: "server", targetId: String(elsa), actorLabel: "system", createdAt: dbTime(NOW.getTime() - 3 * DAY + 2 * HOUR) },
]);
const date = (days: number) => new Date(NOW.getTime() + days * DAY).toISOString().slice(0, 10);
await db.insert(secrets).values([
{ name: "VPN key", type: "api_token", expiryDate: date(9), warnDays: 30 },
{ name: "Old token", type: "api_token", expiryDate: date(-4), warnDays: 30 },
{ name: "Fine for years", type: "api_token", expiryDate: date(900), warnDays: 30 },
]);
await db.insert(maintenanceWindows).values([elsa, anna].map((id) => ({
targetType: "server" as const,
targetId: id,
reason: "Patch night",
startedAt: new Date(NOW.getTime() + 2 * DAY).toISOString(),
endsAt: new Date(NOW.getTime() + 2 * DAY + 2 * HOUR).toISOString(),
createdBy: "Ada",
})));
webhook = await new Promise<http.Server>((resolve) => {
const s = http.createServer((req, res) => {
let raw = "";
req.on("data", (c) => (raw += c));
req.on("end", () => {
sent.push(JSON.parse(raw));
res.end("ok");
});
});
s.listen(0, "127.0.0.1", () => resolve(s));
});
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
next();
});
a.use("/api/settings", settingsRouter);
app = await new Promise<http.Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/settings`;
invalidateAlertsCache();
});
after(async () => {
// Saving the schedule started a cron job, which would keep this process alive.
await (await import("node-schedule")).default.gracefulShutdown();
app.close();
webhook.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("it gathers each topic into a few lines", async () => {
const digest = await buildDigest(NOW);
assert.equal(digest.quiet, false);
const m = digest.message;
assert.match(m, /RIGHT NOW\n.*critical/, m);
assert.ok(!/RIGHT NOW[^]*Secret "Old token"[^]*EXPIRING/.test(m), "an expiring secret is in its own section, not listed twice");
assert.match(m, /\(1 more is about things expiring, below\)|\(2 more are about things expiring, below\)/);
assert.match(m, /elsa hasn't reported for 5 h/);
assert.match(m, /UPTIME, LAST 7 DAYS\n- elsa: .*% \(.*down\)/s, "elsa's downtime is told with a figure");
assert.match(m, /WHAT HAPPENED\n- elsa was offline for 2 h/);
assert.match(m, /EXPIRING\n- Secret "Old token" expired 4 days ago\n- Secret "VPN key" expires in 9 days/);
assert.ok(!m.includes("Fine for years"));
assert.match(m, /UPDATES WAITING\n1 server has updates waiting \(3 security fixes among them\)\.\n1 server needs a restart\.\nLongest wait: anna, 12 d\.\n- anna: 9 waiting, 3 security/);
assert.match(m, /PLANNED MAINTENANCE\n- .*2 h: anna, elsa — Patch night/);
});
test("preview needs an administrator's route and shows the same text; sending needs somewhere to send to", async () => {
const preview = await (await fetch(`${base}/digest/preview`)).json();
assert.match(preview.title, /^Homelab weekly digest/);
assert.match(preview.message, /UPDATES WAITING/);
assert.equal(preview.channels, 0);
const none = await fetch(`${base}/digest/send`, { method: "POST" });
assert.equal(none.status, 400);
assert.equal((await none.json()).error, "no_channel");
assert.equal(sent.length, 0);
});
test("sending goes out through a switched-on channel at once, and is in the audit log", async () => {
const url = `http://127.0.0.1:${(webhook.address() as AddressInfo).port}/hook`;
await updateSettings({ webhook: { enabled: true, url }, quietHours: { enabled: true, start: "00:00", end: "23:59" } });
const res = await fetch(`${base}/digest/send`, { method: "POST" });
assert.equal(res.status, 200);
assert.deepEqual(await res.json(), { sent: true, channels: 1 });
assert.equal(sent.length, 1, "sent at once even inside quiet hours: a person asked for it now");
assert.match(sent[0].content, /^\*\*Homelab weekly digest/);
assert.match(sent[0].content, /UPDATES WAITING/);
const entries = (await db.select().from(auditLog)).filter((e) => e.action === "send_digest");
assert.equal(entries.length, 1);
await updateSettings({ quietHours: { enabled: false } });
});
test("the schedule is saved, and a day or time that makes no sense is refused", async () => {
const put = (digest: unknown) => fetch(`${base}/`, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ digest }) });
assert.equal((await put({ enabled: true, day: 0, time: "21:30" })).status, 200);
assert.deepEqual((await getSettings()).digest, { enabled: true, day: 0, time: "21:30" });
for (const bad of [{ day: 7 }, { day: -1 }, { day: 1.5 }, { time: "25:00" }, { time: "8am" }, { enabled: "yes" }]) {
assert.equal((await put(bad)).status, 400, JSON.stringify(bad));
}
assert.deepEqual((await getSettings()).digest, { enabled: true, day: 0, time: "21:30" }, "nothing was half-saved");
});
test("it is off until switched on", async () => {
const fresh = mkdtempSync(join(tmpdir(), "x-"));
rmSync(fresh, { recursive: true, force: true });
const { digest } = await getSettings();
assert.equal(typeof digest.enabled, "boolean");
});
});