Add a weekly digest and report which version of the agent each server runs
- Weekly digest through the notification channels: alerts now, a week of uptime, what happened, disks heading for full, expiring secrets and domains, updates waiting, planned maintenance. Off until switched on in Settings -> Notifications; Preview and Send it now there. Setting key 'digest'. - Agents say their version in every report (Linux and Windows scripts are version 2); the server compares it with the scripts it serves and marks agents that are behind on the Servers, server and Updates pages. servers.agent_version, migration 0026. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3da5c639b1
commit
aa94d84f6c
28 files changed
+3179
-19
No files matched your search
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* Which version of the agent each server runs, and which agents are behind the scripts this app serves.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-agentver-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { parseAgentVersion, latestAgentVersions, isAgentOutdated } = await import("../src/services/agentVersions.js");
|
||||
const { serversRouter } = await import("../src/routes/servers.js");
|
||||
const { agentReportRouter } = await import("../src/routes/agentReport.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
describe("reading a version out of a script", () => {
|
||||
test("each script says what it is, in the form the server looks for", () => {
|
||||
assert.equal(parseAgentVersion("#!/usr/bin/env bash\nset -e\n\nAGENT_VERSION=7\nENV_FILE=x\n", "linux"), 7);
|
||||
assert.equal(parseAgentVersion("$script:AgentVersion = 12\r\n$script:Other = 1\r\n", "windows"), 12);
|
||||
assert.equal(parseAgentVersion("# AGENT_VERSION=3 in a comment\n", "linux"), null, "only a line that is the assignment counts");
|
||||
assert.equal(parseAgentVersion("no version here", "windows"), null);
|
||||
});
|
||||
|
||||
test("the scripts this app really serves each carry a version", () => {
|
||||
const latest = latestAgentVersions();
|
||||
assert.ok(Number.isInteger(latest.linux) && latest.linux! >= 2, `linux ${latest.linux}`);
|
||||
assert.ok(Number.isInteger(latest.windows) && latest.windows! >= 2, `windows ${latest.windows}`);
|
||||
});
|
||||
|
||||
test("a folder with nothing in it gives no version, not an error", () => {
|
||||
const empty = mkdtempSync(join(tmpdir(), "hm-noagent-"));
|
||||
assert.deepEqual(latestAgentVersions(empty), { linux: null, windows: null });
|
||||
mkdirSync(join(empty, "linux"));
|
||||
writeFileSync(join(empty, "linux", "report-tasks.sh"), "AGENT_VERSION=5\n");
|
||||
assert.deepEqual(latestAgentVersions(empty), { linux: 5, windows: null });
|
||||
rmSync(empty, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("which agents are behind", () => {
|
||||
const latest = { linux: 3, windows: 2 };
|
||||
const seen = "2026-10-17T00:00:00Z";
|
||||
|
||||
test("an agent that says nothing is version 1, and older than the latest is behind", () => {
|
||||
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: null }, latest), true);
|
||||
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 2 }, latest), true);
|
||||
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 3 }, latest), false);
|
||||
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 4 }, latest), false, "ahead of this app is not behind");
|
||||
assert.equal(isAgentOutdated({ osType: "windows", lastSeenAt: seen, agentVersion: 2 }, latest), false, "each system has its own latest");
|
||||
});
|
||||
|
||||
test("a server with no agent has nothing to update, and an unknown latest flags nothing", () => {
|
||||
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: null, agentVersion: null }, latest), false);
|
||||
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: null }, { linux: null, windows: null }), false);
|
||||
assert.equal(isAgentOutdated({ osType: "plan9", lastSeenAt: seen, agentVersion: null }, latest), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the version in reports", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let id = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values({ oidcSub: "admin", name: "Ada", role: "admin" });
|
||||
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t" }).returning();
|
||||
id = s.id;
|
||||
const a = express();
|
||||
a.use("/api/agent/report", agentReportRouter);
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/servers", serversRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const report = (extra: Record<string, unknown>) =>
|
||||
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", tasks: [], ...extra }) });
|
||||
|
||||
test("an agent's version is stored; a report without one, or with nonsense, leaves it as it was", async () => {
|
||||
assert.equal((await report({})).status, 202);
|
||||
let [s] = await db.select().from(servers).where(eq(servers.id, id));
|
||||
assert.equal(s.agentVersion, null, "an old agent says nothing");
|
||||
|
||||
assert.equal((await report({ agent_version: 2 })).status, 202);
|
||||
[s] = await db.select().from(servers).where(eq(servers.id, id));
|
||||
assert.equal(s.agentVersion, 2);
|
||||
|
||||
for (const odd of ["two", 0, -1, 1.5, 999999999, null, {}]) {
|
||||
assert.equal((await report({ agent_version: odd })).status, 202, `${JSON.stringify(odd)} must not cost the report`);
|
||||
}
|
||||
[s] = await db.select().from(servers).where(eq(servers.id, id));
|
||||
assert.equal(s.agentVersion, 2);
|
||||
});
|
||||
|
||||
test("the lists say what the latest is, and the server's own row says what it runs", async () => {
|
||||
const list = await (await fetch(`${base}/servers`)).json();
|
||||
assert.ok(list.agentLatest.linux >= 2 && list.agentLatest.windows >= 2);
|
||||
assert.equal(list.servers.find((x: { id: number }) => x.id === id).agentVersion, 2);
|
||||
|
||||
const detail = await (await fetch(`${base}/servers/${id}/detail`)).json();
|
||||
assert.equal(detail.server.agentVersion, 2);
|
||||
assert.ok(detail.agentLatest.linux >= 2);
|
||||
|
||||
const updates = await (await fetch(`${base}/servers/updates`)).json();
|
||||
assert.equal(updates.servers.find((x: { id: number }) => x.id === id).agentVersion, 2);
|
||||
assert.ok(updates.agentLatest.windows >= 2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,202 @@
|
||||
/**
|
||||
* The weekly digest: what it gathers, what it leaves out, and sending it.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import http from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-digest-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, secrets, auditLog, maintenanceWindows } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { buildDigest, spanWords } = await import("../src/services/weeklyDigest.js");
|
||||
const { cronForDigest } = await import("../src/services/weeklyDigestScheduler.js");
|
||||
const { settingsRouter } = await import("../src/routes/settings.js");
|
||||
const { updateSettings, getSettings } = await import("../src/services/settingsStore.js");
|
||||
const { invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const HOUR = 3600_000;
|
||||
const DAY = 24 * HOUR;
|
||||
const NOW = new Date(Math.floor(Date.now() / 1000) * 1000);
|
||||
const dbTime = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace("T", " ");
|
||||
|
||||
describe("small pieces", () => {
|
||||
test("spans read as words", () => {
|
||||
assert.equal(spanWords(35), "35 min");
|
||||
assert.equal(spanWords(160), "2 h 40 min");
|
||||
assert.equal(spanWords(60 * 30), "1 d 6 h");
|
||||
assert.equal(spanWords(60 * 9), "9 h");
|
||||
assert.equal(spanWords(60 * 24 * 3 + 240), "3 d 4 h");
|
||||
});
|
||||
|
||||
test("the schedule is the chosen weekday and time", () => {
|
||||
assert.equal(cronForDigest(1, "08:00"), "0 8 * * 1");
|
||||
assert.equal(cronForDigest(0, "21:30"), "30 21 * * 0");
|
||||
assert.equal(cronForDigest(9, "nonsense"), "0 8 * * 1", "a bad value falls back rather than scheduling nonsense");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the quiet week", () => {
|
||||
test("with nothing wrong it says so, and leaves out topics with nothing to say", async () => {
|
||||
const digest = await buildDigest(NOW);
|
||||
assert.equal(digest.quiet, true);
|
||||
assert.match(digest.message, /A quiet week: nothing needs attention\./);
|
||||
assert.match(digest.message, /RIGHT NOW\nNo active alerts\./);
|
||||
for (const heading of ["UPTIME", "WHAT HAPPENED", "DISKS HEADING FOR FULL", "EXPIRING", "UPDATES WAITING", "PLANNED MAINTENANCE"]) {
|
||||
assert.ok(!digest.message.includes(heading), `no ${heading} section when there's nothing in it`);
|
||||
}
|
||||
assert.match(digest.title, /^Homelab weekly digest/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a week with things in it", () => {
|
||||
let app: http.Server;
|
||||
let base = "";
|
||||
let webhook: http.Server;
|
||||
const sent: { content: string }[] = [];
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values({ oidcSub: "admin", name: "Ada", role: "admin" });
|
||||
const made = await db
|
||||
.insert(servers)
|
||||
.values([
|
||||
{ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: new Date(NOW.getTime() - 5 * HOUR).toISOString() },
|
||||
{ name: "anna", apiTokenHash: "b", apiTokenPrefix: "b", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: NOW.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 9, security: 3, rebootRequired: true, reportedAt: NOW.toISOString() }), updatesPendingSince: new Date(NOW.getTime() - 12 * DAY).toISOString() },
|
||||
{ name: "zoe", apiTokenHash: "c", apiTokenPrefix: "c", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: NOW.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 0, security: 0, rebootRequired: false, reportedAt: NOW.toISOString() }) },
|
||||
])
|
||||
.returning({ id: servers.id });
|
||||
const [elsa, anna] = made.map((m) => m.id);
|
||||
|
||||
// elsa was down for 2 hours three days ago, and has now stopped reporting again.
|
||||
await db.insert(auditLog).values([
|
||||
{ category: "server", action: "went_offline", targetType: "server", targetId: String(elsa), actorLabel: "system", createdAt: dbTime(NOW.getTime() - 3 * DAY) },
|
||||
{ category: "server", action: "back_online", targetType: "server", targetId: String(elsa), actorLabel: "system", createdAt: dbTime(NOW.getTime() - 3 * DAY + 2 * HOUR) },
|
||||
]);
|
||||
|
||||
const date = (days: number) => new Date(NOW.getTime() + days * DAY).toISOString().slice(0, 10);
|
||||
await db.insert(secrets).values([
|
||||
{ name: "VPN key", type: "api_token", expiryDate: date(9), warnDays: 30 },
|
||||
{ name: "Old token", type: "api_token", expiryDate: date(-4), warnDays: 30 },
|
||||
{ name: "Fine for years", type: "api_token", expiryDate: date(900), warnDays: 30 },
|
||||
]);
|
||||
|
||||
await db.insert(maintenanceWindows).values([elsa, anna].map((id) => ({
|
||||
targetType: "server" as const,
|
||||
targetId: id,
|
||||
reason: "Patch night",
|
||||
startedAt: new Date(NOW.getTime() + 2 * DAY).toISOString(),
|
||||
endsAt: new Date(NOW.getTime() + 2 * DAY + 2 * HOUR).toISOString(),
|
||||
createdBy: "Ada",
|
||||
})));
|
||||
|
||||
webhook = await new Promise<http.Server>((resolve) => {
|
||||
const s = http.createServer((req, res) => {
|
||||
let raw = "";
|
||||
req.on("data", (c) => (raw += c));
|
||||
req.on("end", () => {
|
||||
sent.push(JSON.parse(raw));
|
||||
res.end("ok");
|
||||
});
|
||||
});
|
||||
s.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/settings", settingsRouter);
|
||||
app = await new Promise<http.Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/settings`;
|
||||
invalidateAlertsCache();
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
// Saving the schedule started a cron job, which would keep this process alive.
|
||||
await (await import("node-schedule")).default.gracefulShutdown();
|
||||
app.close();
|
||||
webhook.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("it gathers each topic into a few lines", async () => {
|
||||
const digest = await buildDigest(NOW);
|
||||
assert.equal(digest.quiet, false);
|
||||
const m = digest.message;
|
||||
assert.match(m, /RIGHT NOW\n.*critical/, m);
|
||||
assert.ok(!/RIGHT NOW[^]*Secret "Old token"[^]*EXPIRING/.test(m), "an expiring secret is in its own section, not listed twice");
|
||||
assert.match(m, /\(1 more is about things expiring, below\)|\(2 more are about things expiring, below\)/);
|
||||
assert.match(m, /elsa hasn't reported for 5 h/);
|
||||
assert.match(m, /UPTIME, LAST 7 DAYS\n- elsa: .*% \(.*down\)/s, "elsa's downtime is told with a figure");
|
||||
assert.match(m, /WHAT HAPPENED\n- elsa was offline for 2 h/);
|
||||
assert.match(m, /EXPIRING\n- Secret "Old token" expired 4 days ago\n- Secret "VPN key" expires in 9 days/);
|
||||
assert.ok(!m.includes("Fine for years"));
|
||||
assert.match(m, /UPDATES WAITING\n1 server has updates waiting \(3 security fixes among them\)\.\n1 server needs a restart\.\nLongest wait: anna, 12 d\.\n- anna: 9 waiting, 3 security/);
|
||||
assert.match(m, /PLANNED MAINTENANCE\n- .*2 h: anna, elsa — Patch night/);
|
||||
});
|
||||
|
||||
test("preview needs an administrator's route and shows the same text; sending needs somewhere to send to", async () => {
|
||||
const preview = await (await fetch(`${base}/digest/preview`)).json();
|
||||
assert.match(preview.title, /^Homelab weekly digest/);
|
||||
assert.match(preview.message, /UPDATES WAITING/);
|
||||
assert.equal(preview.channels, 0);
|
||||
|
||||
const none = await fetch(`${base}/digest/send`, { method: "POST" });
|
||||
assert.equal(none.status, 400);
|
||||
assert.equal((await none.json()).error, "no_channel");
|
||||
assert.equal(sent.length, 0);
|
||||
});
|
||||
|
||||
test("sending goes out through a switched-on channel at once, and is in the audit log", async () => {
|
||||
const url = `http://127.0.0.1:${(webhook.address() as AddressInfo).port}/hook`;
|
||||
await updateSettings({ webhook: { enabled: true, url }, quietHours: { enabled: true, start: "00:00", end: "23:59" } });
|
||||
const res = await fetch(`${base}/digest/send`, { method: "POST" });
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(await res.json(), { sent: true, channels: 1 });
|
||||
assert.equal(sent.length, 1, "sent at once even inside quiet hours: a person asked for it now");
|
||||
assert.match(sent[0].content, /^\*\*Homelab weekly digest/);
|
||||
assert.match(sent[0].content, /UPDATES WAITING/);
|
||||
const entries = (await db.select().from(auditLog)).filter((e) => e.action === "send_digest");
|
||||
assert.equal(entries.length, 1);
|
||||
await updateSettings({ quietHours: { enabled: false } });
|
||||
});
|
||||
|
||||
test("the schedule is saved, and a day or time that makes no sense is refused", async () => {
|
||||
const put = (digest: unknown) => fetch(`${base}/`, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ digest }) });
|
||||
assert.equal((await put({ enabled: true, day: 0, time: "21:30" })).status, 200);
|
||||
assert.deepEqual((await getSettings()).digest, { enabled: true, day: 0, time: "21:30" });
|
||||
for (const bad of [{ day: 7 }, { day: -1 }, { day: 1.5 }, { time: "25:00" }, { time: "8am" }, { enabled: "yes" }]) {
|
||||
assert.equal((await put(bad)).status, 400, JSON.stringify(bad));
|
||||
}
|
||||
assert.deepEqual((await getSettings()).digest, { enabled: true, day: 0, time: "21:30" }, "nothing was half-saved");
|
||||
});
|
||||
|
||||
test("it is off until switched on", async () => {
|
||||
const fresh = mkdtempSync(join(tmpdir(), "x-"));
|
||||
rmSync(fresh, { recursive: true, force: true });
|
||||
const { digest } = await getSettings();
|
||||
assert.equal(typeof digest.enabled, "boolean");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user