Add a weekly digest and report which version of the agent each server runs

- Weekly digest through the notification channels: alerts now, a week of uptime, what happened, disks
  heading for full, expiring secrets and domains, updates waiting, planned maintenance. Off until switched
  on in Settings -> Notifications; Preview and Send it now there. Setting key 'digest'.
- Agents say their version in every report (Linux and Windows scripts are version 2); the server compares
  it with the scripts it serves and marks agents that are behind on the Servers, server and Updates pages.
  servers.agent_version, migration 0026.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-17 19:49:36 +02:00
1 parent 3da5c639b1
commit aa94d84f6c
28 files changed
+3179 -19

No files matched your search

+138
View File
@@ -0,0 +1,138 @@
/**
* Which version of the agent each server runs, and which agents are behind the scripts this app serves.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-agentver-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { parseAgentVersion, latestAgentVersions, isAgentOutdated } = await import("../src/services/agentVersions.js");
const { serversRouter } = await import("../src/routes/servers.js");
const { agentReportRouter } = await import("../src/routes/agentReport.js");
const { hashToken } = await import("../src/services/tokens.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
describe("reading a version out of a script", () => {
test("each script says what it is, in the form the server looks for", () => {
assert.equal(parseAgentVersion("#!/usr/bin/env bash\nset -e\n\nAGENT_VERSION=7\nENV_FILE=x\n", "linux"), 7);
assert.equal(parseAgentVersion("$script:AgentVersion = 12\r\n$script:Other = 1\r\n", "windows"), 12);
assert.equal(parseAgentVersion("# AGENT_VERSION=3 in a comment\n", "linux"), null, "only a line that is the assignment counts");
assert.equal(parseAgentVersion("no version here", "windows"), null);
});
test("the scripts this app really serves each carry a version", () => {
const latest = latestAgentVersions();
assert.ok(Number.isInteger(latest.linux) && latest.linux! >= 2, `linux ${latest.linux}`);
assert.ok(Number.isInteger(latest.windows) && latest.windows! >= 2, `windows ${latest.windows}`);
});
test("a folder with nothing in it gives no version, not an error", () => {
const empty = mkdtempSync(join(tmpdir(), "hm-noagent-"));
assert.deepEqual(latestAgentVersions(empty), { linux: null, windows: null });
mkdirSync(join(empty, "linux"));
writeFileSync(join(empty, "linux", "report-tasks.sh"), "AGENT_VERSION=5\n");
assert.deepEqual(latestAgentVersions(empty), { linux: 5, windows: null });
rmSync(empty, { recursive: true, force: true });
});
});
describe("which agents are behind", () => {
const latest = { linux: 3, windows: 2 };
const seen = "2026-10-17T00:00:00Z";
test("an agent that says nothing is version 1, and older than the latest is behind", () => {
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: null }, latest), true);
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 2 }, latest), true);
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 3 }, latest), false);
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 4 }, latest), false, "ahead of this app is not behind");
assert.equal(isAgentOutdated({ osType: "windows", lastSeenAt: seen, agentVersion: 2 }, latest), false, "each system has its own latest");
});
test("a server with no agent has nothing to update, and an unknown latest flags nothing", () => {
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: null, agentVersion: null }, latest), false);
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: null }, { linux: null, windows: null }), false);
assert.equal(isAgentOutdated({ osType: "plan9", lastSeenAt: seen, agentVersion: null }, latest), false);
});
});
describe("the version in reports", () => {
let app: Server;
let base = "";
let id = 0;
before(async () => {
await db.insert(users).values({ oidcSub: "admin", name: "Ada", role: "admin" });
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t" }).returning();
id = s.id;
const a = express();
a.use("/api/agent/report", agentReportRouter);
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
next();
});
a.use("/api/servers", serversRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const report = (extra: Record<string, unknown>) =>
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", tasks: [], ...extra }) });
test("an agent's version is stored; a report without one, or with nonsense, leaves it as it was", async () => {
assert.equal((await report({})).status, 202);
let [s] = await db.select().from(servers).where(eq(servers.id, id));
assert.equal(s.agentVersion, null, "an old agent says nothing");
assert.equal((await report({ agent_version: 2 })).status, 202);
[s] = await db.select().from(servers).where(eq(servers.id, id));
assert.equal(s.agentVersion, 2);
for (const odd of ["two", 0, -1, 1.5, 999999999, null, {}]) {
assert.equal((await report({ agent_version: odd })).status, 202, `${JSON.stringify(odd)} must not cost the report`);
}
[s] = await db.select().from(servers).where(eq(servers.id, id));
assert.equal(s.agentVersion, 2);
});
test("the lists say what the latest is, and the server's own row says what it runs", async () => {
const list = await (await fetch(`${base}/servers`)).json();
assert.ok(list.agentLatest.linux >= 2 && list.agentLatest.windows >= 2);
assert.equal(list.servers.find((x: { id: number }) => x.id === id).agentVersion, 2);
const detail = await (await fetch(`${base}/servers/${id}/detail`)).json();
assert.equal(detail.server.agentVersion, 2);
assert.ok(detail.agentLatest.linux >= 2);
const updates = await (await fetch(`${base}/servers/updates`)).json();
assert.equal(updates.servers.find((x: { id: number }) => x.id === id).agentVersion, 2);
assert.ok(updates.agentLatest.windows >= 2);
});
});
+202
View File
@@ -0,0 +1,202 @@
/**
* The weekly digest: what it gathers, what it leaves out, and sending it.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import http from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-digest-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, secrets, auditLog, maintenanceWindows } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { buildDigest, spanWords } = await import("../src/services/weeklyDigest.js");
const { cronForDigest } = await import("../src/services/weeklyDigestScheduler.js");
const { settingsRouter } = await import("../src/routes/settings.js");
const { updateSettings, getSettings } = await import("../src/services/settingsStore.js");
const { invalidateAlertsCache } = await import("../src/services/alerts.js");
await runMigrations();
const HOUR = 3600_000;
const DAY = 24 * HOUR;
const NOW = new Date(Math.floor(Date.now() / 1000) * 1000);
const dbTime = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace("T", " ");
describe("small pieces", () => {
test("spans read as words", () => {
assert.equal(spanWords(35), "35 min");
assert.equal(spanWords(160), "2 h 40 min");
assert.equal(spanWords(60 * 30), "1 d 6 h");
assert.equal(spanWords(60 * 9), "9 h");
assert.equal(spanWords(60 * 24 * 3 + 240), "3 d 4 h");
});
test("the schedule is the chosen weekday and time", () => {
assert.equal(cronForDigest(1, "08:00"), "0 8 * * 1");
assert.equal(cronForDigest(0, "21:30"), "30 21 * * 0");
assert.equal(cronForDigest(9, "nonsense"), "0 8 * * 1", "a bad value falls back rather than scheduling nonsense");
});
});
describe("the quiet week", () => {
test("with nothing wrong it says so, and leaves out topics with nothing to say", async () => {
const digest = await buildDigest(NOW);
assert.equal(digest.quiet, true);
assert.match(digest.message, /A quiet week: nothing needs attention\./);
assert.match(digest.message, /RIGHT NOW\nNo active alerts\./);
for (const heading of ["UPTIME", "WHAT HAPPENED", "DISKS HEADING FOR FULL", "EXPIRING", "UPDATES WAITING", "PLANNED MAINTENANCE"]) {
assert.ok(!digest.message.includes(heading), `no ${heading} section when there's nothing in it`);
}
assert.match(digest.title, /^Homelab weekly digest/);
});
});
describe("a week with things in it", () => {
let app: http.Server;
let base = "";
let webhook: http.Server;
const sent: { content: string }[] = [];
before(async () => {
await db.insert(users).values({ oidcSub: "admin", name: "Ada", role: "admin" });
const made = await db
.insert(servers)
.values([
{ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: new Date(NOW.getTime() - 5 * HOUR).toISOString() },
{ name: "anna", apiTokenHash: "b", apiTokenPrefix: "b", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: NOW.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 9, security: 3, rebootRequired: true, reportedAt: NOW.toISOString() }), updatesPendingSince: new Date(NOW.getTime() - 12 * DAY).toISOString() },
{ name: "zoe", apiTokenHash: "c", apiTokenPrefix: "c", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: NOW.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 0, security: 0, rebootRequired: false, reportedAt: NOW.toISOString() }) },
])
.returning({ id: servers.id });
const [elsa, anna] = made.map((m) => m.id);
// elsa was down for 2 hours three days ago, and has now stopped reporting again.
await db.insert(auditLog).values([
{ category: "server", action: "went_offline", targetType: "server", targetId: String(elsa), actorLabel: "system", createdAt: dbTime(NOW.getTime() - 3 * DAY) },
{ category: "server", action: "back_online", targetType: "server", targetId: String(elsa), actorLabel: "system", createdAt: dbTime(NOW.getTime() - 3 * DAY + 2 * HOUR) },
]);
const date = (days: number) => new Date(NOW.getTime() + days * DAY).toISOString().slice(0, 10);
await db.insert(secrets).values([
{ name: "VPN key", type: "api_token", expiryDate: date(9), warnDays: 30 },
{ name: "Old token", type: "api_token", expiryDate: date(-4), warnDays: 30 },
{ name: "Fine for years", type: "api_token", expiryDate: date(900), warnDays: 30 },
]);
await db.insert(maintenanceWindows).values([elsa, anna].map((id) => ({
targetType: "server" as const,
targetId: id,
reason: "Patch night",
startedAt: new Date(NOW.getTime() + 2 * DAY).toISOString(),
endsAt: new Date(NOW.getTime() + 2 * DAY + 2 * HOUR).toISOString(),
createdBy: "Ada",
})));
webhook = await new Promise<http.Server>((resolve) => {
const s = http.createServer((req, res) => {
let raw = "";
req.on("data", (c) => (raw += c));
req.on("end", () => {
sent.push(JSON.parse(raw));
res.end("ok");
});
});
s.listen(0, "127.0.0.1", () => resolve(s));
});
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
next();
});
a.use("/api/settings", settingsRouter);
app = await new Promise<http.Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/settings`;
invalidateAlertsCache();
});
after(async () => {
// Saving the schedule started a cron job, which would keep this process alive.
await (await import("node-schedule")).default.gracefulShutdown();
app.close();
webhook.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("it gathers each topic into a few lines", async () => {
const digest = await buildDigest(NOW);
assert.equal(digest.quiet, false);
const m = digest.message;
assert.match(m, /RIGHT NOW\n.*critical/, m);
assert.ok(!/RIGHT NOW[^]*Secret "Old token"[^]*EXPIRING/.test(m), "an expiring secret is in its own section, not listed twice");
assert.match(m, /\(1 more is about things expiring, below\)|\(2 more are about things expiring, below\)/);
assert.match(m, /elsa hasn't reported for 5 h/);
assert.match(m, /UPTIME, LAST 7 DAYS\n- elsa: .*% \(.*down\)/s, "elsa's downtime is told with a figure");
assert.match(m, /WHAT HAPPENED\n- elsa was offline for 2 h/);
assert.match(m, /EXPIRING\n- Secret "Old token" expired 4 days ago\n- Secret "VPN key" expires in 9 days/);
assert.ok(!m.includes("Fine for years"));
assert.match(m, /UPDATES WAITING\n1 server has updates waiting \(3 security fixes among them\)\.\n1 server needs a restart\.\nLongest wait: anna, 12 d\.\n- anna: 9 waiting, 3 security/);
assert.match(m, /PLANNED MAINTENANCE\n- .*2 h: anna, elsa — Patch night/);
});
test("preview needs an administrator's route and shows the same text; sending needs somewhere to send to", async () => {
const preview = await (await fetch(`${base}/digest/preview`)).json();
assert.match(preview.title, /^Homelab weekly digest/);
assert.match(preview.message, /UPDATES WAITING/);
assert.equal(preview.channels, 0);
const none = await fetch(`${base}/digest/send`, { method: "POST" });
assert.equal(none.status, 400);
assert.equal((await none.json()).error, "no_channel");
assert.equal(sent.length, 0);
});
test("sending goes out through a switched-on channel at once, and is in the audit log", async () => {
const url = `http://127.0.0.1:${(webhook.address() as AddressInfo).port}/hook`;
await updateSettings({ webhook: { enabled: true, url }, quietHours: { enabled: true, start: "00:00", end: "23:59" } });
const res = await fetch(`${base}/digest/send`, { method: "POST" });
assert.equal(res.status, 200);
assert.deepEqual(await res.json(), { sent: true, channels: 1 });
assert.equal(sent.length, 1, "sent at once even inside quiet hours: a person asked for it now");
assert.match(sent[0].content, /^\*\*Homelab weekly digest/);
assert.match(sent[0].content, /UPDATES WAITING/);
const entries = (await db.select().from(auditLog)).filter((e) => e.action === "send_digest");
assert.equal(entries.length, 1);
await updateSettings({ quietHours: { enabled: false } });
});
test("the schedule is saved, and a day or time that makes no sense is refused", async () => {
const put = (digest: unknown) => fetch(`${base}/`, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ digest }) });
assert.equal((await put({ enabled: true, day: 0, time: "21:30" })).status, 200);
assert.deepEqual((await getSettings()).digest, { enabled: true, day: 0, time: "21:30" });
for (const bad of [{ day: 7 }, { day: -1 }, { day: 1.5 }, { time: "25:00" }, { time: "8am" }, { enabled: "yes" }]) {
assert.equal((await put(bad)).status, 400, JSON.stringify(bad));
}
assert.deepEqual((await getSettings()).digest, { enabled: true, day: 0, time: "21:30" }, "nothing was half-saved");
});
test("it is off until switched on", async () => {
const fresh = mkdtempSync(join(tmpdir(), "x-"));
rmSync(fresh, { recursive: true, force: true });
const { digest } = await getSettings();
assert.equal(typeof digest.enabled, "boolean");
});
});